{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Otto商品分类——LightGBM\n",
    "原始特征+tfidf特征\n",
    "boosting_type参数取值goss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们以Kaggle 2015年举办的Otto Group Product Classification Challenge竞赛数据为例，LightGBM以进行参数调优。\n",
    "\n",
    "Otto数据集是著名电商Otto提供的一个多类商品分类问题，类别数=9. 每个样本有93维数值型特征（整数，表示某种事件发生的次数，已经进行过脱敏处理）。 竞赛官网：https://www.kaggle.com/c/otto-group-product-classification-challenge/data\n",
    "\n",
    "\n",
    "第一名：https://www.kaggle.com/c/otto-group-product-classification-challenge/discussion/14335\n",
    "第二名：http://blog.kaggle.com/2015/06/09/otto-product-classification-winners-interview-2nd-place-alexander-guschin/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import lightgbm as lgbm\n",
    "from lightgbm.sklearn import LGBMClassifier\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>feat_1_tfidf</th>\n",
       "      <th>feat_2_tfidf</th>\n",
       "      <th>feat_3_tfidf</th>\n",
       "      <th>feat_4_tfidf</th>\n",
       "      <th>feat_5_tfidf</th>\n",
       "      <th>feat_6_tfidf</th>\n",
       "      <th>feat_7_tfidf</th>\n",
       "      <th>feat_8_tfidf</th>\n",
       "      <th>feat_9_tfidf</th>\n",
       "      <th>...</th>\n",
       "      <th>feat_85_tfidf</th>\n",
       "      <th>feat_86_tfidf</th>\n",
       "      <th>feat_87_tfidf</th>\n",
       "      <th>feat_88_tfidf</th>\n",
       "      <th>feat_89_tfidf</th>\n",
       "      <th>feat_90_tfidf</th>\n",
       "      <th>feat_91_tfidf</th>\n",
       "      <th>feat_92_tfidf</th>\n",
       "      <th>feat_93_tfidf</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.081393</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.075886</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.231403</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.199730</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.011987</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011668</td>\n",
       "      <td>0.105971</td>\n",
       "      <td>0.021681</td>\n",
       "      <td>0.080435</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.008244</td>\n",
       "      <td>0.022456</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.124622</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.145988</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 95 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  feat_1_tfidf  feat_2_tfidf  feat_3_tfidf  feat_4_tfidf  feat_5_tfidf  \\\n",
       "0   1      0.081393           0.0           0.0      0.000000      0.000000   \n",
       "1   2      0.000000           0.0           0.0      0.000000      0.000000   \n",
       "2   3      0.000000           0.0           0.0      0.000000      0.000000   \n",
       "3   4      0.011987           0.0           0.0      0.011668      0.105971   \n",
       "4   5      0.000000           0.0           0.0      0.000000      0.000000   \n",
       "\n",
       "   feat_6_tfidf  feat_7_tfidf  feat_8_tfidf  feat_9_tfidf  ...  feat_85_tfidf  \\\n",
       "0      0.000000      0.000000      0.000000           0.0  ...       0.075886   \n",
       "1      0.000000      0.000000      0.231403           0.0  ...       0.000000   \n",
       "2      0.000000      0.000000      0.199730           0.0  ...       0.000000   \n",
       "3      0.021681      0.080435      0.000000           0.0  ...       0.000000   \n",
       "4      0.000000      0.000000      0.000000           0.0  ...       0.124622   \n",
       "\n",
       "   feat_86_tfidf  feat_87_tfidf  feat_88_tfidf  feat_89_tfidf  feat_90_tfidf  \\\n",
       "0       0.000000       0.000000            0.0            0.0       0.000000   \n",
       "1       0.000000       0.000000            0.0            0.0       0.000000   \n",
       "2       0.000000       0.000000            0.0            0.0       0.000000   \n",
       "3       0.008244       0.022456            0.0            0.0       0.000000   \n",
       "4       0.000000       0.000000            0.0            0.0       0.145988   \n",
       "\n",
       "   feat_91_tfidf  feat_92_tfidf  feat_93_tfidf   target  \n",
       "0            0.0            0.0            0.0  Class_1  \n",
       "1            0.0            0.0            0.0  Class_1  \n",
       "2            0.0            0.0            0.0  Class_1  \n",
       "3            0.0            0.0            0.0  Class_1  \n",
       "4            0.0            0.0            0.0  Class_1  \n",
       "\n",
       "[5 rows x 95 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "# 这里使用tf_idf特征\n",
    "train = pd.read_csv(\"./Otto_FE_train_tfidf.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将类别字符串变成数字，LightGBM不支持字符串格式的特征输入/标签输入\n",
    "y_train = train['target'] #形式为Class_x\n",
    "y_train = y_train.map(lambda s: s[6:])\n",
    "y_train = y_train.map(lambda s: int(s) - 1)#将类别的形式由Class_x变为0-8之间的整数\n",
    "\n",
    "X_train = train.drop([\"id\", \"target\"], axis=1)\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns \n",
    "\n",
    "#sklearn的学习器大多之一稀疏数据输入，模型训练会快很多\n",
    "#查看一个学习器是否支持稀疏数据，可以看fit函数是否支持: X: {array-like, sparse matrix}.\n",
    "#可自行用timeit比较稠密数据和稀疏数据的训练时间\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LightGBM超参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LightGBM的主要的超参包括：\n",
    "1. 树的数目n_estimators 和 学习率 learning_rate\n",
    "2. 树的最大深度max_depth 和 树的最大叶子节点数目num_leaves（注意：XGBoost只有max_depth，LightGBM采用叶子优先的方式生成树，num_leaves很重要，设置成比 2^max_depth 小）\n",
    "3. 叶子结点的最小样本数:min_data_in_leaf(min_data, min_child_samples)\n",
    "4. 每棵树的列采样比例：feature_fraction/colsample_bytree\n",
    "5. 每棵树的行采样比例：bagging_fraction （需同时设置bagging_freq=1）/subsample\n",
    "6. 正则化参数lambda_l1(reg_alpha), lambda_l2(reg_lambda)\n",
    "\n",
    "7. 两个非模型复杂度参数，但会影响模型速度和精度。可根据特征取值范围和样本数目修改这两个参数\n",
    "1）特征的最大bin数目max_bin：默认255；\n",
    "2）用来建立直方图的样本数目subsample_for_bin：默认200000。\n",
    "\n",
    "对n_estimators，用LightGBM内嵌的cv函数调优，因为同XGBoost一样，LightGBM学习的过程内嵌了cv，速度极快。\n",
    "其他参数用GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_ROUNDS = 10000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 相同的交叉验证分组"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. n_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#直接调用lightgbm内嵌的交叉验证(cv)，可对连续的n_estimators参数进行快速交叉验证\n",
    "#而GridSearchCV只能对有限个参数进行交叉验证，且速度相对较慢\n",
    "def get_n_estimators(params , X_train , y_train , early_stopping_rounds=10):\n",
    "    lgbm_params = params.copy()\n",
    "    lgbm_params['num_class'] = 9\n",
    "     \n",
    "    lgbmtrain = lgbm.Dataset(X_train , y_train )\n",
    "     \n",
    "    #num_boost_round为弱分类器数目，下面的代码参数里因为已经设置了early_stopping_rounds\n",
    "    #即性能未提升的次数超过过早停止设置的数值，则停止训练\n",
    "    cv_result = lgbm.cv(lgbm_params , lgbmtrain , num_boost_round=MAX_ROUNDS , nfold=3,  metrics='multi_logloss' , early_stopping_rounds=early_stopping_rounds,seed=3 )\n",
    "     \n",
    "    print('best n_estimators:' , len(cv_result['multi_logloss-mean']))\n",
    "    print('best cv score:' , cv_result['multi_logloss-mean'][-1])\n",
    "     \n",
    "    return len(cv_result['multi_logloss-mean'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best n_estimators: 328\n",
      "best cv score: 0.4995384287882351\n"
     ]
    }
   ],
   "source": [
    "params = {'boosting_type': 'goss',\n",
    "          'objective': 'multiclass',\n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          'num_leaves': 60,\n",
    "          'max_depth': 6,\n",
    "          'max_bin': 127, #2^6,原始特征为整数，很少超过100\n",
    "          'colsample_bytree': 0.7,\n",
    "         }\n",
    "\n",
    "n_estimators_1 = get_n_estimators(params , X_train , y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. num_leaves & max_depth=7\n",
    "num_leaves建议70-80，搜索区间50-80,值越大模型越复杂，越容易过拟合\n",
    "相应的扩大max_depth=7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done   8 out of  12 | elapsed:  6.7min remaining:  3.4min\n",
      "[Parallel(n_jobs=4)]: Done  12 out of  12 | elapsed: 10.3min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "             error_score=nan,\n",
       "             estimator=LGBMClassifier(boosting_type='goss', class_weight=None,\n",
       "                                      colsample_bytree=0.7,\n",
       "                                      importance_type='split',\n",
       "                                      learning_rate=0.1, max_bin=127,\n",
       "                                      max_depth=7, min_child_samples=20,\n",
       "                                      min_child_weight=0.001,\n",
       "                                      min_split_gain=0.0, n_estimators=328,\n",
       "                                      n_jobs=4, num_class=9, num_leaves=31,\n",
       "                                      objective='multiclass', random_state=None,\n",
       "                                      reg_alpha=0.0, reg_lambda=0.0,\n",
       "                                      silent=False, subsample=1.0,\n",
       "                                      subsample_for_bin=200000,\n",
       "                                      subsample_freq=0),\n",
       "             iid='deprecated', n_jobs=4,\n",
       "             param_grid={'num_leaves': range(50, 90, 10)},\n",
       "             pre_dispatch='2*n_jobs', refit=False, return_train_score=False,\n",
       "             scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'goss',\n",
    "          'objective': 'multiclass',\n",
    "          'num_class':9, \n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':n_estimators_1,\n",
    "          'max_depth': 7,\n",
    "          'max_bin': 127, #2^6,原始特征为整数，很少超过100\n",
    "          'colsample_bytree': 0.7,\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "num_leaves_s = range(50,90,10) #50,60,70,80\n",
    "tuned_parameters = dict( num_leaves = num_leaves_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=4, param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , y_train)\n",
    "#grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5001304071628401\n",
      "{'num_leaves': 80}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEHCAYAAABm9dtzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXhU5fn/8fedhH2TSJAtyCKiiIgSEtS27ogbLlhlUUFFrC31p7ZW7aLWpV+32lVtAQW0sihiBa0i1qVuBMK+iQQQiGxhlUWWJPfvjznRMSZkgBlmJvm8rutczJxznjPP4zH55CxzbnN3REREoiEl3h0QEZGqQ6EiIiJRo1AREZGoUaiIiEjUKFRERCRq0uLdgXhq0qSJt2nTJt7dEBFJKjNnztzo7hnlLavWodKmTRvy8vLi3Q0RkaRiZisrWqbTXyIiEjUKFRERiRqFioiIRI1CRUREokahIiIiUaNQERGRqFGoiIhI1ChUpMp4b8kGlqzbHu9uiFRr1frLj1J1PPVePo9PWQJAt6Mb0z+7NRd1aU7tGqlx7plI9aIjFUl6//hgGY9PWcKlXVvwmwuPZ8vOvfzi5bnk/OG/PDB5EfkbdsS7iyLVho5UJKmN+HA5j7z5GZec1II//vgk0lJTGPzDtny6fBNjclfxwrQveO7jFWS3TWdATmt6dW5GrTQdvYjEikJFktazH63goTcWc1GX5vzpqlCgAJgZp7Vvwmntm7Bxxx5ezitg7PRV/L9xc0ivV5Mru7WiX3Zr2japF+cRiFQ9Vp1r1GdlZbkeKJmcRn28gvsnL+KCzs34a7+TqZG6/zO5JSXOx8s2MiZ3FW8vWk9xiXNa+yPpn9Oanp2aUTNNZ4JFImVmM909q7xlMf1JMrNeZrbEzPLN7O5ylg8ys0IzmxNMg8OWDTSzpcE0MGz+w2a22sy+d6LczK4ys0VmttDMxsRuZBJPL3z6BfdPXkTPTkdFFCgAKSnGDztk8Mw13fj07rP5Zc9jWblpF0PHzOa0R/7Lo299xqpNu2LfeZEqLmZHKmaWCnwOnAcUADOAfu6+KGydQUCWuw8t0zYdyAOyAAdmAt3cfYuZ9QBWAkvdvX5Ymw7AS8DZwXpN3X3D/vqoI5Xk82LuSn7z6gLOPf4onh5wyiEdYRSXOP9bWsiY3FX8d/F6Shx+2KEJA3Jac87xR0UUViLV0f6OVGJ5TSUbyHf35UEnxgGXAov22yrkfGCqu28O2k4FegFj3X1aMK9sm5uAp9x9C0BlgSLJZ9z0Vfzm1QWcfVxTnhpw8iGfskpNMc7q2JSzOjZl7bavGT9jNeNnrOYn/5pF0wa1uLp7Jld3z6RV47pRGoFI1RfLP8VaAqvD3hcE88rqY2bzzGyCmWUeYNtwxwLHmtnHZjbNzHqVt5KZDTGzPDPLKywsjGwkEncv5a3mnlfnc2bHDJ655pSo38HVvFEdbjv3WD781VmMuC6LE1o05O/v5fPDx97j+pHTeWfReoqKS6L6mSJVUSyPVL53KEHoVFa4yYSOPvaY2U+A0cDZEbYtKw3oAJwJtAI+NLPO7r71OxtxHwYMg9Dpr8oGIfE3YWYBd70yjx8c04R/XNMtprcEp6WmcG6nozi301EUbNn1zdHL4OfzaN6o9jdHL80b1YlZH0SSWSyPVAqAzLD3rYA14Su4+yZ33xO8HQ50i7RtBZ/3mrvvc/cVwBJCISNJ7NXZBdw5YS6nt2/C8OuyDus35Fs1rssvenbk47vP5h/XdOOYpvX58ztLOf2Rdxk8Oo/3lmyguER/l4iEi+WRygygg5m1Bb4E+gL9w1cws+buvjZ42xtYHLyeAvzBzBoH73sC91Tyef8G+gGjzKwJodNhyw95FBI3r835kl+8NJdT2x152AMlXI3UFHp1bkavzs1YtWkXY2es4uW81byzeD0tj6hDv+xMrsrKpGnD2nHpn0giien3VMzsQuDPQCrwnLs/bGYPAHnuPsnM/o9QmBQBm4Fb3P2zoO0NwK+DTT3s7iOD+Y8RCqcWhI5eRrj7/Ra6cv9HQhf0i4M24/bXP939lbhen7eGW8fOpnubdEZe3526NRPre7p7i0qYumg9L+au5JNlm0hLMc49/igG9GjN6e2bkJJS3hlckaphf3d/6cuPCpWE85/5a/n52Nl0a92Ykdd3p16txAqUslZs3MnY6aGjly279tE6vS79slvz46xWNKlfK97dE4k6hUoFFCqJ560F6xg6ZhZdM49g1A3Z1E/wQAm3p6iYtxas48XcVUxfsZkaqUbPE5oxIKc1p7Y7srzb4EWSkkKlAgqVxPL2wnX89MVZdGnViNE3ZNOgdo14d+mg5W/Yzpjc1bwyq4BtX++jXZN69MtuTZ9urUivVzPe3RM5JAqVCihUEsd/F6/nJ/+aSacWjXjhxmwaJnGghNu9r5j/zF/Li7mrmLlyCzVTU7jgxGYMyDma7m0a6+hFkpJCpQIKlcTw3mcbuPmFmRzXvAEv3JhDozpVI1DKWrJuO2NyVzJx9pds313EMU3r0z+7NX1OaUWjulVzzFI1KVQqoFCJvw8+L+Sm5/M49qj6vHhjj2rxy3XX3iJenxc6epm7eiu10lK4qEtzBuQczSmtj9DRiyQ8hUoFFCrx9dHSjdwwegbHZNRnzE05HFG3+l1rWLhmG2NyV/Hv2V+yc28xxzVrQP+c1lx2cssqcwpQqh6FSgUUKvHzSf5Grh81g7ZN6jH2ph40ruYXr3fuKWLS3DW8mLuSBV9+RZ0aqVxyUujopUurRjp6kYSiUKmAQiU+Pl22ietHTefo9HqMuSmHI/Vdju+YV7CVMbmreG3OGr7eV8wJLRrSP6c1l3ZtmVS3WEvVpVCpgELl8MtdvolBI2fQqnEdxg7poS8H7sdXu/fx2uwveTF3FZ+t2069mqlcenJL+me3pnPLRvHunlRjCpUKKFQOrxlfbGbgc9Np3qg244acSkYDBUok3J3Zq0NHL6/PW8PufSWc1KoR/XNac8lJLRLuETZS9SlUKqBQOXxmrtzCdc/mclTD2owb0kMPXzxI23btY+LsAsbkrmLphh00qJXG5ae0pH9Oa45r1jDe3ZNqQqFSAYXK4TF71RaufXY6GQ1qMW5ID45SoBwydydv5RbG5K7ijflr2VtUwimtj6B/ztFc3KV53J7oLNWDQqUCCpXYm7t6K9eMyCW9fk3GDzmVZo0UKNG2ZedeXpkVOnpZvnEnDWun0adbKwbktOaYpg3i3T2pghQqFVCoxNb8gm0MGDGNRnVrMH7IqbQ4QtUSY8ndmbZ8M2Omr+KtBWvZV+xkt0mnf05renVupqMXiRqFSgUUKrGz4MttDBiRS4PaaYwb0oNWjevGu0vVysYde5gws4Cx01exctMuGtetwZXdWtEvuzXtMurHu3uS5BQqFVCoxMaiNV/Rf8Q06tUMBUpmugIlXkpKnE+WbWLM9JW8vXA9RSXOqe2OpH9Oa84/oRk102JZUVyqKoVKBRQq0ffZuq/oN2watWukMn7IqbQ+UoGSKDZs383LeaGjl4ItX3NkvZr8OCuTftmZHH1kvXh3T5LI/kIlpn+mmFkvM1tiZvlmdnc5yweZWaGZzQmmwWHLBprZ0mAaGDb/YTNbbWY7KvjMK83MzazcAUvsfL5+OwOG51IrLZWxN/VQoCSYpg1q87OzjuF/d57FqOu70+3oxgz/cDlnPP4+1z6by5vz17KvuCTe3ZQkF7MjFTNLBT4HzgMKgBlAP3dfFLbOICDL3YeWaZsO5AFZgAMzgW7uvsXMegArgaXuXr9MuwbAG0BNYKi77/cwREcq0ZO/YTt9h00jxYzxN59K2yb6yzcZrNu2m/EzVjN+xirWbNtNRoNaXJ2VSd/sTF0HkwrF60glG8h39+XuvhcYB1waYdvzganuvtndtwBTgV4A7j7N3ddW0O5B4DFg96F1XQ7EssId9Buei5kxdkgPBUoSadaoNv/v3A58eNfZPDswiy4tG/H0+/n88LH3GDRyOm8vXEeRjl7kAMTy+Q4tgdVh7wuAnHLW62NmPyJ0VHO7u6+uoG3L/X2YmZ0MZLr762b2y/2sNwQYAtC6detIxiH7sbxwB/2GTcMdxg3Job3uLEpKqSnGOccfxTnHH8WXW7/+5uhlyAszadawNld3Dx29NG+k28Jl/2J5pFLes7rLnmubDLRx9y7AO8DoA2j77QeZpQB/An5RWafcfZi7Z7l7VkZGRmWry358sXEn/YZPo7jEGXNTjr5oV0W0PKIOd5x3LB/fdTb/vLYbHZs14K/vLuX0R95l8OgZvPvZeopLqu8NPrJ/sTxSKQAyw963AtaEr+Dum8LeDgceDWt7Zpm27+/nsxoAnYH3g7oTzYBJZta7susqcnBWbgoFyr7iUKAce5QCpapJS03h/BOacf4JzVi9eRfjZqxi/IwC3lmcR8sj6tC3eyZXdc/UY3fkO2J5oT6N0Cmtc4AvCV2o7+/uC8PWaV56fcTMLgfucvcewYX6mcApwaqzCF2o3xzWdkfZC/Vhy94HfqkL9bGxevMurv7np+zaV8yYwT3o1EIPMqwu9hWXMHXResbkruKj/I2kphjnHt+U/jlH88NjmpCSomJi1cH+LtTH7EjF3YvMbCgwBUgFnnP3hWb2AJDn7pOAW82sN1AEbAYGBW03m9mDhIII4IHSQDGzx4D+QF0zKwBGuPv9sRqHfFfBll30HTaNnXuLGXNTjgKlmqmRmsKFJzbnwhOb88XGnYydsYqX8wqYsnA9rdPr0jc7kx93y1RZg2pMX37UkUrEvtz6NX2Hfcq2XfsYc1MPFYoSAPYUFTNl4XrG5K5k2vLN1Eg1enZqRv+c1pza7kgdvVRBcTlSkapl7bav6TdsGlt37eNfN+YoUOQbtdJS6X1SC3qf1IJlhTsYm7uKCbMKeGP+Wto2qcfNP2pH32zdaVld6ME/Uqn1X+2m37BpbNm5l+dvyOakzCPi3SVJUO0z6vPbizsx7Z5z+NPVJ9GwTg3unjifD5cWxrtrcpgoVGS/NgSBUrh9D6NuyObk1o3j3SVJArVrpHL5ya0YP6QH7ZrU456J89m5pyje3ZLDQKEiFdqwfTf9hk9j3Ve7GX1DNt2OVqDIgaldI5VHr+xCwZav+ePbn8e7O3IYKFSkXBt37GHA8FzWbN3NqOuzyWqTHu8uSZLq3iada3sczchPVjBr1ZZ4d0diTKEi37Npxx76D5/G6i27eG5Qd7LbKlDk0PyqV0eaN6zNXRPmsaeoON7dkRhSqMh3bN65lwEjclm5aRfPDezOqe2PjHeXpApoULsGD19+Iks37ODp95bFuzsSQwoV+cbWXXu5ZkQuKzbu5NmB3TntmCbx7pJUIWcd15TLurbg6ffz+WzdV/HujsSIQkUA2LZrHwNG5JJfuIPh12Xxgw4KFIm+ey85gQa1a3DXK/P1UMoqSqEibPt6H9c8m8vS9Tv457Xd+NGxenqzxEZ6vZrc3/sE5q7eysiPV8S7OxIDCpVq7qvd+7juuel8tu4rnrnmFM7q2DTeXZIq7pIuzTnnuKY88fYSVm3aFe/uSJQpVKqx7bv3MfC56Sxas42nB3TjnOOPineXpBowMx66vDNpKSncPXEe1fn5g1WRQqWa2rGniEEjZzC/YBt/63cK53VSoMjh07xRHe658Dg+WbaJl/MK4t0diSKFSjW0c08R14+czpzVW/lbv5Pp1blZvLsk1VC/7q3JbpvOg28sYv1Xu+PdHYkShUo1s2tvEdePmsGsVVv5S9+uXHBi83h3SaqplBTjkStOZG9RCfe+tiDe3ZEoUahUI1/vLebGUXnkfbGZP13dlYu7tIh3l6Saa5dRn9vPO5YpC9fz5vy18e6ORIFCpZrYva+Ym57PI3fFJp68qiu9T1KgSGIY/IO2dG7ZkN+9tpCtu/bGuztyiGIaKmbWy8yWmFm+md1dzvJBZlZoZnOCaXDYsoFmtjSYBobNf9jMVpvZjjLbusPMFpnZPDP7r5kdHcuxJZPSQPl42UYev/IkLju5Zby7JPKNtNQUHu3ThS279vLQG4vj3R05RDELFTNLBZ4CLgA6Af3MrFM5q453967BNCJomw7cB+QA2cB9Zlb63PXJwbyyZgNZ7t4FmAA8FtUBJand+4q5+YWZfJS/kUf7dKFPt1bx7pLI95zQohE/OaMdE2YWqKBXkovlkUo2kO/uy919LzAOuDTCtucDU919s7tvAaYCvQDcfZq7f+/kq7u/5+6l36SaBlT73557ior56Yuz+ODzQh654kSuysqMd5dEKvTzszvQLkMFvZJdLEOlJbA67H1BMK+sPsEpqwlmVvpbL9K2FbkReLO8BWY2xMzyzCyvsLDq/kW0t6iEn704i3c/28AfLj+Rq7urRrgktto1Unm0T6ig1xNvL4l3d+QgxTJUrJx5Zb86OxloE5yyegcYfQBty/9Qs2uALODx8pa7+zB3z3L3rIyMqvmMq33FJQwdM4t3Fm/gwcs60z9HgSLJoXubdK479WhGffKFCnolqViGSgEQfr6lFbAmfAV33+Tue4K3w4FukbYtj5mdC/wG6B223WplX3EJPx8zm7cXref3vU/g2h66X0GSy696HaeCXkkslqEyA+hgZm3NrCbQF5gUvoKZhX/zrjdQeuvHFKCnmTUOLtD3DOZVyMxOBv5JKFA2RGkMSaWouITbxs3hrYXruPfiTgw8rU28uyRywOrXSvumoNdTKuiVdGIWKu5eBAwlFAaLgZfcfaGZPWBmvYPVbjWzhWY2F7gVGBS03Qw8SCiYZgAPBPMws8fMrACoa2YFZnZ/sK3HgfrAy8Htyd8JsKquqLiE21+ayxvz1/Lbi47nhh+0jXeXRA7aWcc15fKTW/KMCnolHavOTwjNysryvLy8eHfjkBWXOHe8NIfX5qzhnguO4+Yz2se7SyKHbPPOvZz75AdkNq7DxJ+eTmpKeZdaJR7MbKa7Z5W3TN+oT3LFJc6dL8/ltTlr+FWvjgoUqTK+KehVsE0FvZKIQiWJlZQ4d70yj4mzv+QX5x3LT888Jt5dEomqS7o059zjVdArmShUklRJiXPPxPlMmFnAbed24OfndIh3l0Sizsx48DIV9EomCpUkVFLi/ObfCxift5pbzz6G2849Nt5dEomZ8IJeL+WtrryBxJVCJcm4O/dOWsDY6av42Vntuf08BYpUff26tyanbToPvbFYBb0SnEIlibg7901ayL+mreLmM9rxy54dMdMdMVL1paQYj/Tpwt6iEn737wU6DZbAFCpJwt35/eRFPP/pSm76YVvu7nWcAkWqlbZN6nH7ecfy9qL1vLlgXby7IxVQqCQBd+ehNxYz6pMvuOH0tvz6wuMVKFItlRb0ulcFvRKWQiXBuTuPvPkZz360gkGnteF3FytQpPpSQa/Ep1BJYO7OY1OW8M//LefaHkdz3yWdFChS7YUX9Prf51W3fEWyUqgkKHfnj29/zjPvL6N/Tmt+3/sEBYpIoLSg169fVUGvRKNQSVB/fmcpf38vn77dM3no0s6k6LlHIt9QQa/EpVBJQH/971L+8t+lXJXVij9cfqICRaQc4QW9Zq5UQa9EUWmomFl7M6sVvD7TzG41syNi37Xq6an38nly6uf0OaUVj1zRRYEish+lBb3ufkUFvRJFJEcqrwDFZnYM8CzQFhgT015VU8+8v4zHpyzh8pNb8tiVChSRytSvlcbDV6igVyKJJFRKgoJblwN/dvfbgeaVtJEDNOx/y3j0rc/ofVILnvjxSaodIRKhszqGCno9/Z4KeiWCSEJln5n1AwYCrwfzakSycTPrZWZLzCzfzO4uZ/kgMysMKjXOMbPBYcsGmtnSYBoYNv9hM1ttZjvKbKuWmY0PPivXzNpE0sdEMOLD5fzhP59xcZfmPHmVAkXkQP3u4k40qlODuybMo7hEj3CJp0hC5XrgVOBhd19hZm2Bf1XWyMxSgaeAC4BOQD8z61TOquPdvWswjQjapgP3ATlANnBfUKseYHIwr6wbgS3ufgzwJ+DRCMYWdyM/XsFDbyzmwhOb8eeru5KWqnsnRA5Uer2a3KeCXgmh0t9g7r7I3W9197HBL/YG7v5IBNvOBvLdfbm77wXGAZdG2K/zganuvtndtwBTgV5Bf6a5+9py2lwKjA5eTwDOsQT/Ysfzn37B7ycv4vwTjuIvfU9WoIgcgvCCXis37Yx3d6qtSO7+et/MGgZHD3OBkWb2ZATbbgmEFz8oCOaV1cfM5pnZBDPLPMC25X5ecA1oG3BkBP2Mi39NW8m9ry3kvE5H8bd+p1BDgSJySEoLetVISeGeifP1JOM4ieQ3WSN3/wq4Ahjp7t2AcyNoV95RQtm9PBlo4+5dgHf49kgjkrYH83mY2RAzyzOzvMLC+DziYez0Vfz23ws457imPNX/FGqmKVBEoqF5ozrcrYJecRXJb7M0M2sOXMW3F+ojUQBkhr1vBawJX8HdN7n7nuDtcKBbpG3393lmlgY0AjaXXcndh7l7lrtnZWRkRDiU6HlpxmrumTifszpm8PQ1ChSRaFNBr/iK5DfaA8AUYJm7zzCzdsDSCNrNADqYWVszqwn0BSaFrxCEVaneQOljR6cAPc2scXAdp2cwb38mEbpDDeBK4F1PsOPfCTMLuGviPH50bAbPXNONWmmp8e6SSJWjgl7xFcmF+pfdvYu73xK8X+7ufSJoVwQMJRQGi4GX3H2hmT1gZr2D1W41s4VmNhe4FRgUtN0MPEgomGYADwTzMLPHzKwAqGtmBWZ2f7CtZ4EjzSwfuAP43i3M8TRxVgF3TpjLD45pwrBru1G7hgJFJFZU0Ct+rLIUN7NWwN+A0wldo/gI+H/uXhD77sVWVlaW5+XlxfxzXpvzJbePn0OPdkfy7MDu1KmpQBGJtaLiEi57+mPWbdvNO3ecwRF1a8a7S1WGmc1096zylkVy+mskoVNLLQjdYTU5mCcRmDx3DbePn0N223QFishhlJaawmN9TmLrrn08+LoKeh0ukYRKhruPdPeiYBoFHP4r3EnojXlruW38HLKOTue5QQoUkcOtU4uG3HxGO16ZpYJeh0skobLRzK4xs9RgugbYFOuOJbu3Fqzl1nGzOTnzCEZe3526NdPi3SWRaqm0oNc9E1XQ63CIJFRuIHQ78TpgLaE7q66PZaeS3ZSF6xg6ZjYntWrEqBuyqVdLgSISL7VrpPJYny6s2fY1j09RQa9Yi+Tur1Xu3tvdM9y9qbtfRuiLkFKOdxatZ+iYWXRu2YjRN2RTX4EiEndZbdK5tsfRjP5UBb1i7WC/eXdHVHtRRbz32QZ++uIsOjVvyPM3ZtOgdkQPcxaRw6C0oNddKugVUwcbKgn9oMZ4eH/JBm5+YSYdmzXg+RtzaKhAEUkopQW98jfs4Kl38+PdnSrrYENFX1EN87/PCxnywkyOaVqfF27MplEdBYpIIvqmoNf7y1i8VgW9YqHCUDGz7Wb2VTnTdkLfWRHg4/yN3PR8Hu0z6vPi4Bx9wUokwX1T0OuVeRQVl8S7O1VOhaHi7g3cvWE5UwN319Vn4JNlG7lx9AzaNqnHi4NzaFxPgSKS6NLr1eT+3icwr2AbIz/+It7dqXL0iNyDNG35Jm4clUfr9Lq8ODiHdAWKSNK4OCjo9cepKugVbQqVgzB9xWZuGDWDlo3r8OLgHhxZv1a8uyQiByC8oNfdr6igVzQpVA7Cmq1f0/KIOoy5KYeMBgoUkWTUvFEd7rnweD5dvonxM1TQK1oUKgfhspNb8satP6Rpg9rx7oqIHIK+3TPJaZvOw/9RQa9oiaRGfXl3ga02s1eDgl3Vkio2iiS/8IJev1VBr6iI5Dfjk8CdhB573wr4JaHSv+OA52LXNRGR2GvbpB53nHcsUxet5z/zVdDrUEUSKr3c/Z/uvt3dv3L3YcCF7j4eaBzj/omIxNyNP2jLiS0bcd+kBWzdtTfe3UlqkYRKiZldZWYpwXRV2LL9HiuaWS8zW2Jm+Wb2vfK+ZjbIzArNbE4wDQ5bNtDMlgbTwLD53cxsfrDNv5qZBfO7mtm0YDt5ZpYdwdhEREhLTeHRPl1U0CsKIgmVAcC1wIZguha4xszqEKpBXy4zSwWeAi4AOgH9zKxTOauOd/euwTQiaJsO3AfkANnAfWZWelT0DDAE6BBMvYL5jwG/d/euwL3BexGRiHRq0ZCfnNGeV2YV8IEKeh20SB59v9zdL3H3JsF0ibvnu/vX7v7RfppmA/lB+72ErsFcGmG/zgemuvtmd98CTAV6mVlzoKG7f+qhK2rPA5eVdhVoGLxuBKyJ8LNERAAYevYxtMuox69V0OugRXL3V6vgTq8NZrbezF4xs1YRbLslEH7zd0Ewr6w+ZjbPzCaYWWYlbVsGr8vb5m3A42a2GngCuKeC8QwJTo/lFRbqrxER+ZYKeh26SE5/jQQmEXqIZEtgcjCvMuU9Hr/sNZjJQBt37wK8A4yupO3+tnkLcLu7ZwK3A8+W1yl3H+buWe6elZGRUckQRKS6yWqTznUq6HXQIgmVDHcf6e5FwTQKiOS3cQGQGfa+FWVOSbn7JnffE7wdDnSrpG1B8Lq8bQ4EJgavXyZ0+k1E5IDdqYJeBy2SUNloZteYWWowXQNsiqDdDKCDmbU1s5pAX0JHPN8IrpGU6g2U3nYxBehpZo2DC/Q9gSnuvhbYbmY9gru+rgNeC9qsAc4IXp8NLI2gjyIi36OCXgcvkkfY3wD8HfgToVNNnwDXV9bI3YvMbCihgEgFnnP3hWb2AJDn7pOAW82sN1AEbAYGBW03m9mDhIIJ4AF33xy8vgUYBdQB3gwmgJuAv5hZGrCb0B1iIiIH5ayOTbkiKOh1wYnNOb55w8obCXYwjyUws9vc/c8x6M9hlZWV5Xl5efHuhogkqC0793Lukx/QsnEdJt5yGmmpejwTgJnNdPes8pYd7H+hOw6hPyIiSaGxCnodsIMNlfLuwhIRqXJCBb2OUkGvCB1sqOhRniJSLZgZD6mgV8QqDJUKHnn/lZltJ/SdFRGRaqFZo9oq6BWhCkPF3Ru4e8NypgbuHsldYyIiVUbf7pn0aKeCXpXRrQwiIhFISTEeuUIFvSqjUBERiVAbFVkRZIUAABB/SURBVPSqlEJFROQAhBf02rJTBb3KUqiIiByA8IJeD72hgl5lKVRERA6QCnpVTKEiInIQhp59DO1V0Ot7FCoiIgehdo1UHlVBr+9RqIiIHKTvFvTaXOn61YFCRUTkENzZ6zhaNKrDXa/MV0EvFCoiIoekfq00Hr68swp6BRQqIiKH6Mywgl6L134V7+7ElUJFRCQKfndxJxrVqcFdr8yjqLgk3t2Jm5iGipn1MrMlZpZvZneXs3yQmRWa2ZxgGhy2bKCZLQ2mgWHzu5nZ/GCbfw1q1Zcu+3nweQvN7LFYjk1EJJwKeoXELFTMLBV4CrgA6AT0M7NO5aw63t27BtOIoG06cB+QA2QD95lZ42D9ZwjVn+8QTL2CNmcBlwJd3P0E4IlYjU1EpDzhBb2+2Fg9C3rF8kglG8h39+XuvhcYR+iXfiTOB6a6+2Z33wJMBXqZWXOgobt/6qFHhD4PXBa0uQV4xN33ALj7hmgORkSkMuEFve6ZWD0LesUyVFoC4dVsCoJ5ZfUxs3lmNsHMMitp2zJ4Xd42jwV+aGa5ZvaBmXUvr1NmNsTM8swsr7BQj1cQkeiq7gW9Yhkq5dWxLxvbk4E27t4FeAcYXUnb/W0zDWgM9ADuBF4Kv97yzcruw9w9y92zMjIyKh+FiMgB+qag1xuLWbetehX0imWoFACZYe9bAWvCV3D3TaWnq4DhQLdK2hYEr8vbZgEw0UOmAyVAkyiMQ0TkgHxT0Ku4hN+9Vr0KesUyVGYAHcysrZnVBPoCk8JXCK6RlOoNlD5HegrQ08waBxfoewJT3H0tsN3MegRHIdcBrwVt/g2cHWz3WKAmsDE2QxMR2b/qWtArZqHi7kXAUEIBsRh4yd0XmtkDZtY7WO3W4PbfucCtwKCg7WbgQULBNAN4IJgHoQvyI4B8YBnwZjD/OaCdmS0gdFPAQK9Ofx6ISMKpjgW9rDr/3s3KyvK8vLx4d0NEqrBFa76i998/onfXFjx5Vdd4dycqzGymu2eVt0zfqBcRiaHSgl4TZ31ZLQp6KVRERGIsvKDXjipe0EuhIiISY+EFvZ6o4gW9FCoiIodBdSnopVARETlMSgt6/WrCPHbvq5oFvRQqIiKHSWlBr2WFO3nqvapZ0EuhIiJyGJUW9Hqmihb0UqiIiBxmVbmgl0JFROQwCy/o9dzHK+LdnahSqIiIxEFpQa8np35epQp6KVREROIgvKDX3RPnVZknGStURETipLSg17TlmxlXRQp6KVREROKotKDXH6pIQS+FiohIHIUX9Prtv5O/oJdCRUQkzkoLer2zeD1vzF8b7+4cEoWKiEgCKC3odf+khUld0EuhIiKSANJSU3i0Txe27trHg28sind3DlpMQ8XMepnZEjPLN7O7y1k+yMwKzWxOMA0OWzbQzJYG08Cw+d3MbH6wzb8GterDt/lLM3MzaxLLsYmIRFt4Qa/3l2yId3cOSsxCxcxSgaeAC4BOQD8z61TOquPdvWswjQjapgP3ATlANnCfmTUO1n8GGAJ0CKZeYZ+ZCZwHrIrNqEREYqu0oNdvXl2QlAW9Ynmkkg3ku/tyd98LjAMujbDt+cBUd9/s7luAqUAvM2sONHT3Tz10i8TzwGVh7f4E/ApI7tsnRKTaSvaCXrEMlZZA+Ld5CoJ5ZfUxs3lmNiE40thf25bB6+9t08x6A1+6+9z9dcrMhphZnpnlFRZW/XrRIpJ8wgt65X2RXAW9YhkqVs68skcQk4E27t4FeAcYXUnbcuebWV3gN8C9lXXK3Ye5e5a7Z2VkZFS2uohIXJQW9LrrleQq6BXLUCkAMsPetwLWhK/g7pvcfU/wdjjQrZK2BcHrsvPbA22BuWb2RTB/lpk1i8pIREQOs2Qt6BXLUJkBdDCztmZWE+gLTApfIbhGUqo3sDh4PQXoaWaNgwv0PYEp7r4W2G5mPYK7vq4DXnP3+e7e1N3buHsbQuFziruvi+H4RERiKryg16I1yVHQK2ah4u5FwFBCAbEYeMndF5rZA8H1D4BbzWyhmc0FbgUGBW03Aw8SCqYZwAPBPIBbgBFAPrAMeDNWYxARibdkK+hlyf6cmUORlZXleXl58e6GiMh+vT5vDUPHzObXFx7HkB+1j3d3MLOZ7p5V3jJ9o15EJMFddGKooNcf3078gl4KFRGRBFda0KtmauIX9FKoiIgkgWaNavPrixK/oJdCRUQkSSRDQS+FiohIkjBL/IJeChURkSTSpkk9ftEzcQt6KVRERJLMDaeHCnrd91riFfRSqIiIJJnSgl7bvt7Hg68nVkEvhYqISBLq1KIht5zZnomzE6ugl0JFRCRJJWJBL4WKiEiSqpX2bUGvx9/6LN7dARQqIiJJLatNOgNPbcPz01YmREEvhYqISJK78/yOCVPQS6EiIpLk6oUV9Pr7u/Et6KVQERGpAs7s2JQrTmnJPz6Ib0EvhYqISBXxu4viX9BLoSIiUkU0rleT3196AvO/3MazH62ISx9iGipm1svMlphZvpndXc7yQWZWaGZzgmlw2LKBZrY0mAaGze9mZvODbf41qFWPmT1uZp+Z2Twze9XMjojl2EREEtFFJzbnvE5H8eTU+BT0ilmomFkq8BRwAdAJ6GdmncpZdby7dw2mEUHbdOA+IAfIBu4zs8bB+s8AQ4AOwdQrmD8V6OzuXYDPgXtiMzIRkcRlZjx4aaig112vzKOk5PA+yTiWRyrZQL67L3f3vcA44NII254PTHX3ze6+hVBg9DKz5kBDd//UQ898fh64DMDd33b30q+UTgNaRXMwIiLJorSgV+6Kw1/QK5ah0hIIH01BMK+sPsEpqwlmlllJ25bB68q2eQPwZnmdMrMhZpZnZnmFhYWRjUREJMn07Z7Jqe2O5P/+c3gLesUyVKyceWWPwyYDbYJTVu8AoytpW+k2zew3QBHwYnmdcvdh7p7l7lkZGRn76b6ISPIyM/7vihODgl7zD1tBr1iGSgGQGfa+FbAmfAV33+Tue4K3w4FulbQt4Luntb6zzeCC/sXAAE/EkmgiIofRtwW9NvD6vMNT0CuWoTID6GBmbc2sJtAXmBS+QnCNpFRvYHHwegrQ08waBxfoewJT3H0tsN3MegR3fV0HvBZsqxdwF9Db3XfFcFwiIknjhtPb0qVVI+6fdHgKesUsVIKL5kMJBcRi4CV3X2hmD5hZ72C1W81soZnNBW4FBgVtNwMPEgqmGcADwTyAW4ARQD6wjG+vnfwdaABMDW5P/kesxiYikiwOd0Evq85nibKysjwvLy/e3RARibk/vr2Ev72bz6jru3Nmx6aHtC0zm+nuWeUt0zfqRUSqgcNV0EuhIiJSDdRKS+WxK2Nf0EuhIiJSTXQ7OvYFvRQqIiLVyJ3nd6Rzi0Zs3bUvJttPi8lWRUQkIdWrlcakoacTPIs36nSkIiJSzcQqUEChIiIiUaRQERGRqFGoiIhI1ChUREQkahQqIiISNQoVERGJGoWKiIhETbV+SrGZFQIrD7J5E2BjFLsTTxpL4qkq4wCNJVEdyliOdvdyS+dW61A5FGaWV9Gjn5ONxpJ4qso4QGNJVLEai05/iYhI1ChUREQkahQqB29YvDsQRRpL4qkq4wCNJVHFZCy6piIiIlGjIxUREYkahYqIiESNQiVCZvaFmc03szlmlhfMSzezqWa2NPi3cbz7WZkKxnG/mX0ZzJtjZhfGu5+RMLMjzGyCmX1mZovN7NRk3CdQ4ViSbr+YWcew/s4xs6/M7LZk2y/7GUfS7RMAM7vdzBaa2QIzG2tmtc2srZnlBvtkvJnVjMpn6ZpKZMzsCyDL3TeGzXsM2Ozuj5jZ3UBjd78rXn2MRAXjuB/Y4e5PxKtfB8PMRgMfuvuI4AeiLvBrkmyfQIVjuY0k3C+lzCwV+BLIAX5GEu4X+N44rifJ9omZtQQ+Ajq5+9dm9hLwH+BCYKK7jzOzfwBz3f2ZQ/08HakcmkuB0cHr0cBlcexLtWJmDYEfAc8CuPted99KEu6T/Ywl2Z0DLHP3lSThfgkTPo5klQbUMbM0Qn+wrAXOBiYEy6O2TxQqkXPgbTObaWZDgnlHuftagODfpnHrXeTKGwfAUDObZ2bPJfqpiUA7oBAYaWazzWyEmdUjOfdJRWOB5Nsv4foCY4PXybhfSoWPA5Jsn7j7l8ATwCpCYbINmAlsdfeiYLUCoGU0Pk+hErnT3f0U4ALgZ2b2o3h36CCVN45ngPZAV0L/0/0xjv2LVBpwCvCMu58M7ATujm+XDlpFY0nG/QJAcAqvN/ByvPtyKMoZR9LtkyD4LgXaAi2AeoR+/suKyrUQhUqE3H1N8O8G4FUgG1hvZs0Bgn83xK+HkSlvHO6+3t2L3b0EGE5obImuAChw99zg/QRCv5iTbp9QwViSdL+UugCY5e7rg/fJuF+gzDiSdJ+cC6xw90J33wdMBE4DjghOhwG0AtZE48MUKhEws3pm1qD0NdATWABMAgYGqw0EXotPDyNT0ThKf9gDlxMaW0Jz93XAajPrGMw6B1hEku0TqHgsybhfwvTju6eMkm6/BL4zjiTdJ6uAHmZW18yMb39W3gOuDNaJ2j7R3V8RMLN2hP6qh9CpijHu/rCZHQm8BLQmtON+7O6b49TNSu1nHC8QOpx34Avg5tLz34nMzLoCI4CawHJCd+akkET7pFQFY/kryblf6gKrgXbuvi2Yl1Q/K1DhOJL1Z+X3wNVAETAbGEzoGso4ID2Yd4277znkz1KoiIhItOj0l4iIRI1CRUREokahIiIiUaNQERGRqFGoiIhI1ChUREQkahQqIgnEzEaZ2ZWVrymSmBQqIiISNQoVkUqYWZugcNbwoNDR22ZWx8zeN7OsYJ0mQa0azGyQmf3bzCab2QozG2pmdwRPIJ5mZukRfm43M/sgeKL0lLBnZ91kZjPMbK6ZvRI8fqORhQqwpQTr1DWz1WZWw8zam9lbwXY+NLPjgnV+HBRtmmtm/4vJfzypdhQqIpHpADzl7icAW4E+lazfGehP6IGDDwO7gicQfwpcV9mHmVkN4G/Ale7eDXgu2A6ECit1d/eTgMXAjcFjROYCZwTrXAJMCR4gOAz4ebCdXwJPB+vcC5wfbKd3ZX0SiURa5auICKGnvM4JXs8E2lSy/nvuvh3YbmbbgMnB/PlAlwg+ryOhYJoaegYgqYQetQ7Q2cweAo4A6gNTgvnjCT3f6T1CNUCeNrP6hJ5I+3KwHYBawb8fA6OCSoATI+iTSKUUKiKRCX/QXjFQh9DD+UqP9mvvZ/2SsPclRPZzZ8BCdz+1nGWjgMvcfa6ZDQLODOZPAv4vOL3WDXiXUO2Mre7etexG3P0nZpYDXATMMbOu7r4pgr6JVEinv0QO3heEfnnDt48Qj5YlQIaZnQqh02FmdkKwrAGwNjhFNqC0gbvvAKYDfwFeD+p+fAWsMLMfB9sxMzspeN3e3XPd/V5gI5AZ5TFINaRQETl4TwC3mNknQJNobtjd9xIKqkfNbC4wh9BpLIDfAbnAVOCzMk3HA9cE/5YaANwYbGchoSqAAI+b2XwzWwD8j9A1GZFDokffi4hI1OhIRUREokYX6kXiwMyeAk4vM/sv7j4yHv0RiRad/hIRkajR6S8REYkahYqIiESNQkVERKJGoSIiIlHz/wEASkT11/izNAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid_search.cv_results_[ 'std_test_score' ]\n",
    "\n",
    "n_leafs = len(num_leaves_s)\n",
    "\n",
    "x_axis = num_leaves_s\n",
    "plt.plot(x_axis, -test_means)\n",
    "#plt.errorbar(x_axis, -test_means, yerr=test_stds,label = ' Test')\n",
    "#plt.errorbar(x_axis, -train_means, yerr=train_stds,label = ' Train')\n",
    "plt.xlabel( 'num_leaves' )\n",
    "plt.ylabel( 'Log Loss' )\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.50086499, -0.50166008, -0.50134358, -0.50013041])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_means"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 扩大范围"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 3 candidates, totalling 9 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done   4 out of   9 | elapsed:  3.9min remaining:  4.9min\n",
      "[Parallel(n_jobs=4)]: Done   6 out of   9 | elapsed:  7.5min remaining:  3.7min\n",
      "[Parallel(n_jobs=4)]: Done   9 out of   9 | elapsed:  8.6min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "             error_score=nan,\n",
       "             estimator=LGBMClassifier(boosting_type='goss', class_weight=None,\n",
       "                                      colsample_bytree=0.7,\n",
       "                                      importance_type='split',\n",
       "                                      learning_rate=0.1, max_bin=127,\n",
       "                                      max_depth=7, min_child_samples=20,\n",
       "                                      min_child_weight=0.001,\n",
       "                                      min_split_gain=0.0, n_estimators=328,\n",
       "                                      n_jobs=4, num_class=9, num_leaves=31,\n",
       "                                      objective='multiclass', random_state=None,\n",
       "                                      reg_alpha=0.0, reg_lambda=0.0,\n",
       "                                      silent=False, subsample=1.0,\n",
       "                                      subsample_for_bin=200000,\n",
       "                                      subsample_freq=0),\n",
       "             iid='deprecated', n_jobs=4,\n",
       "             param_grid={'num_leaves': range(80, 110, 10)},\n",
       "             pre_dispatch='2*n_jobs', refit=False, return_train_score=False,\n",
       "             scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'goss',\n",
    "          'objective': 'multiclass',\n",
    "          'num_class':9, \n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':n_estimators_1,\n",
    "          'max_depth': 7,\n",
    "          'max_bin': 127, #2^6,原始特征为整数，很少超过100\n",
    "          'colsample_bytree': 0.7,\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "num_leaves_s = range(80,110,10) \n",
    "tuned_parameters = dict( num_leaves = num_leaves_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=4, param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , y_train)\n",
    "#grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.49917791056018007\n",
      "{'num_leaves': 90}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEHCAYAAABm9dtzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXhU5fn/8fdN2CFAIAlC2LcAgkASBfcoWtEqUFcQrH5r9VcUaV3a0lptq22tWndRa/2q/VZEsVbFpUW04r6FfZGwLwEkYUf2hPv3xzmBiBMSYCaT5fO6rrkyc87znLnnZGbuec5yH3N3REREoqFWvAMQEZHqQ0lFRESiRklFRESiRklFRESiRklFRESipna8A4in5ORk79ChQ7zDEBGpUqZNm7be3VMizavRSaVDhw7k5OTEOwwRkSrFzFaUNk+bv0REJGqUVEREJGqUVEREJGqUVEREJGqUVEREJGqUVEREJGqUVEREJGqUVI5A/tZdPP3RsniHISJS6SipHIEXvlzFHW/M51/T8+IdiohIpaKkcgSuy+7MgE7N+fUrc1jw9dZ4hyMiUmkoqRyB2gm1eHh4P5rUr8Oo56azddfeeIckIlIpKKkcodTE+jx6eQYrN+7gFy/NRpdlFhFRUjkqJ3RszthB3fnPvK/5X+24FxFRUjlaPz61I4OOPYa7/r2AL5dvjHc4IiJxpaRylMyMey45jrZJDbh+/HQKtu2Od0giInGjpBIFTerX4fGRmWzdtZcxE2ZQWLQv3iGJiMSFkkqU9GjVhD8M7c2nSzdw/5SF8Q5HRCQulFSi6OLMNgw/oS2PTV3CO/PXxTscEZEKp6QSZb+94Fh6pTXhpokzWblhR7zDERGpUEoqUVa/TgKPj8gE4Lrnp7Frb1GcIxIRqThKKjHQtnlDHrisL3NXb+X3r8+LdzgiIhVGSSVGBvZoyXXZnZnwxSpeylkV73BERCqEkkoM3XR2N07s1ILfvDqX+WtUeFJEqj8llRgqLjzZrGEdrhs/TYUnRaTaU1KJsZTEeoy7PIO8TTu5ZeIsFZ4UkWpNSaUCZHVozthzu/P2/HX87cOl8Q5HRCRmlFQqyNWndOS83sdw939y+XzphniHIyISE0oqFcTMuPui42jfvCGjJ8wgf9uueIckIhJ1SioVKLF+HR4bmcG2XXu54XkVnhSR6kdJpYJ1P6YJf/pBbz5ftpG/vK3CkyJSvSipxMGFGW24vH87nnh/CVNUeFJEqhEllTi5/fye9E5ryk0TZ7Jiw/Z4hyMiEhUxTSpmNsjMcs1ssZmNjTD/KjMrMLOZ4e3HJeZdaWaLwtuVJaZnmtmccJkPm5mF05ub2ZSw/RQzS4rlazta9esk8NiIDGqZMeq56So8KSLVQsySipklAOOAc4GewHAz6xmh6Yvu3je8PRX2bQ78FugPnAD8tkSSeBy4Fuga3gaF08cC77p7V+Dd8HGlFhSe7MP8tVu5/bW58Q5HROSoxXKkcgKw2N2Xuvse4AVgSDn7ngNMcfeN7r4JmAIMMrNWQBN3/9SDU9P/Dxga9hkC/D28//cS0yu1M7u3ZPQZXZiYk8fEL1V4UkSqtlgmlTSg5LdkXjjtYBeZ2Wwz+6eZtS2jb1p4P9IyW7r7WoDwb2qkoMzsWjPLMbOcgoKCw31NMXHj2d04uUsLbnttLvPWbIl3OCIiRyyWScUiTDu48NXrQAd3Pw54hwMjjdL6lmeZh+TuT7p7lrtnpaSkHE7XmEmoZTw0rB9JDesy6rnpbNmpwpMiUjXFMqnkAW1LPG4DrCnZwN03uPvu8OHfgMwy+uaF9yMtc124eYzwb34UXkOFSW5cj3EjMlizeSe3vKTCkyJSNcUyqXwJdDWzjmZWFxgGTCrZoDgJhAYDX4X3JwPfM7OkcAf994DJ4WatbWY2IDzq64fAa2GfSUDxUWJXlpheZWS2T+LX5/Vgyvx1/PUDFZ4UkaqndqwW7O6FZjaaIEEkAE+7+zwzuwPIcfdJwBgzGwwUAhuBq8K+G83sToLEBHCHu28M748CngUaAP8ObwB/Biaa2dXASuCSWL22WPqfkzswbeUm7vnPAvq2bcaATi3iHZKISLlZTd7MkpWV5Tk5OfEO4zu+2V3I4Ec/YuvOQt4acwqpTerHOyQRkf3MbJq7Z0WapzPqK6HG9WrzxMhMtu8uZPQEFZ4UkapDSaWS6tYykbsu7M0XyzZy7+TceIcjIlIuSiqV2NB+aYwc0I6/frCUyfO+jnc4IiJlUlKp5G47vyd92jTllomzWL5ehSdFpHJTUqnk6tVOYNyIDBISjJ88N42de1R4UkQqLyWVKqBNUkMeuKwvueu2cdtrc3VipIhUWkoqVcQZ6anccEYX/jktjxdVeFJEKikllSrkp2d149Suydw+aR5zV6vwpIhUPkoqVUhx4ckWjeoyavw0tuxQ4UkRqVyUVKqY5o3qMm5EBl9v2cXNL81k3z7tXxGRykNJpQrKaJfEref14J2v8nnigyXxDkdEZD8llSrqypM6cEGf1vxlci6fLFkf73BERAAllSrLzPjzhb3pmNyIMRNmsG7rrniHJCKipFKVNQoLT+7YU8To56ezV4UnRSTOlFSquK5h4ckvlwfXYBERiScllWpgSN80fnhie/724TL+M3dtvMMRkRpMSaWauPX7PejTthm3vDSbpQXfxDscEamhlFSqiXq1E3hsRAZ1Eozrxk9X4UkRiQsllWokrVkDHhzWj9x127j11TkqPCkiFU5JpZo5vVsKY87syr+mr2bCFyo8KSIVS0mlGhozsCundUvhd5PmMSdPhSdFpOIoqVRDCbWMBy/rS3LjoPDk5h174h2SiNQQSirVVPNGdXlsZCbrtu7ipomzVHhSRCqEkko11rdtM247vyf/XZDP4++r8KSIxJ6SSjV3xYD2DO7TmvvezuXjxSo8KSKxpaRSzZkZd13Ym04pjRkzYQZfb1HhSRGJHSWVGiAoPJnBzr1FXK/CkyISQ0oqNUSX1ETuvug4pq3YxF1vqfCkiMSGkkoNckGf1lx1Ugee/ngZb85W4UkRiT4llRrm1+f1oF+7Zvzin7NYosKTIhJlSio1TN3atRh3eQb16iQw6rlp7NhTGO+QRKQaUVKpgVo3a8BDw/qyKP8bbn1lrgpPikjUKKnUUKd2TeHGs7rxyozVjP98ZbzDEZFqIqZJxcwGmVmumS02s7GHaHexmbmZZYWP65rZM2Y2x8xmmVl2ibaXmdlsM5tnZveUmN7OzN4zsxnh/PNi+dqqg9FndCE7PYU7Xp/P7LzN8Q5HRKqBmCUVM0sAxgHnAj2B4WbWM0K7RGAM8HmJydcAuHtv4GzgPjOrZWYtgHuBge5+LNDSzAaGfX4DTHT3fsAw4LHYvLLqo1Yt44FL+5KSWI9Rz01n03YVnhSRoxPLkcoJwGJ3X+rue4AXgCER2t0J3AOUPNW7J/AugLvnA5uBLKATsNDdC8J27wAXhfcdaBLebwqsid5Lqb6SGtXlsREZFGzbzY0TZ6rwpIgclVgmlTSg5FWi8sJp+5lZP6Ctu79xUN9ZwBAzq21mHYFMoC2wGOhuZh3MrDYwNJwO8DtgpJnlAW8BN0QKysyuNbMcM8spKCiI1KTG6dO2Gbdd0JOpuQWMe29xvMMRkSoslknFIkzb/zPYzGoBDwA3R2j3NEESygEeBD4BCt19EzAKeBH4EFgOFB8TOxx41t3bAOcB/wif49sBuD/p7lnunpWSknKEL636Gdm/HUP7tub+dxby0SIVnhSRIxPLpJLHgVEEQBu+vUkqEegFTDWz5cAAYJKZZbl7obvf6O593X0I0AxYBODur7t7f3c/Ecgtng5cDUwM23wK1AeSY/bqqhkz408X9qZramPGvDCDtVt2xjskEamCYplUvgS6mllHM6tLsPN8UvFMd9/i7snu3sHdOwCfAYPdPcfMGppZIwAzO5tglDI/fJwa/k0CrgOeChe5EhgYzutBkFS0feswNKxbm8dHZrJ7bxHXjZ/OnkIVnhSRwxOzpOLuhcBoYDLwFcGRWfPM7A4zG1xG91Rgupl9BfwSuKLEvIfMbD7wMfBnd18YTr8ZuMbMZgETgKtcZ/Udts4pjbnn4j7MWLmZP731VbzDEZEqxmry925WVpbn5OTEO4xK6fevz+OZj5fzyPB+XNCndbzDEZFKxMymuXtWpHk6o14i+tW5Pcho14yxL89mcb4KT4pI+SipSER1a9di3IgDhSe371bhSREpm5KKlKpV0wY8PKwfiwu+4devzFHhSREpk5KKHNIpXZO5+exuvDZzDc99tiLe4YhIJVdmUjGzzmZWL7yfbWZjzKxZ7EOTyuK67C6c2T2VO96Yz8xVKjwpIqUrz0jlZaDIzLoA/wt0BJ6PaVRSqdSqZdx/aR9aNqnP9eNVeFJESleepLIvPOfkB8CD7n4j0Cq2YUll06zhgcKTP3tRhSdFJLLyJJW9ZjYcuBIoLvxYJ3YhSWV1XJtm/HZwT95fWMAj/1XhSRH5rvIklf8BTgT+6O7LwqrBz8U2LKmsLj+hHRf2S+PBdxfywUJVwRGRbyszqbj7fHcf4+4Twnpbie7+5wqITSohM+OPP+hNt9REfvrCDFZvVuFJETmgPEd/TTWzJmbWnOA6J8+Y2f2xD00qqwZ1E3h8ZAZ7i5zrVXhSREooz+avpu6+FbgQeMbdM4GzYhuWVHadUhpzz8XHMXPVZv745vx4hyMilUR5kkptM2sFXMqBHfUinNe7FVef0pG/f7qCSbN09WYRKV9SuYOgfP0Sd//SzDpx4MJYUsONPbc7We2TGPvybBat2xbvcEQkzsqzo/4ldz/O3UeFj5e6+0WxD02qgjoJtXj08gwa1k1g1PjpKjwpUsOVZ0d9GzN7xczyzWydmb1sZm0qIjipGo5pWp+Hh/VjacE3jP2XCk+K1GTl2fz1DMFlgFsDacDr4TSR/U7qkszN30vn9Vlr+L9PVXhSpKYqT1JJcfdn3L0wvD0LpMQ4LqmCRp3emYHdU/nDm/OZvnJTvMMRkTgoT1JZb2YjzSwhvI0ENsQ6MKl6gsKTfTmmaX1Gj5/ORhWeFKlxypNUfkRwOPHXwFrgYoLSLSLf0bRhHR4fkcn67Xv46QszKFLhSZEapTxHf61098HunuLuqe4+lOBESJGIeqU15feDj+XDRet56F0dfS5SkxzplR9vimoUUu0MO74tF2W04ZH/LmJqbn68wxGRCnKkScWiGoVUO2bGH4b2Ir1lIj97cSZ5m3bEOyQRqQBHmlS0oVzKFBSezKQoLDy5u7Ao3iGJSIyVmlTMbJuZbY1w20ZwzopImTomN+LeS45jVt4W/vDGV/EOR0RirNSk4u6J7t4kwi3R3WtXZJBStQ3q1YprTu3IPz5bwWszV8c7HBGJoSPd/CVyWH4xqDvHd0hi7MtzWKjCkyLVlpKKVIjiwpON6tXmJ89N4xsVnhSplpRUpMK0bFKfR4b3Y/n67fzy5dkqPClSDSmpSIU6sXMLfn5Od96cvZZnP1ke73BEJMrKU/o+0lFgq8Jy+J0qIkipXn5yeifO6tGSP775FdNWqPCkSHVSnpHK/cDPCcretwFuAf4GvAA8HbvQpLoyM+67tA+tmzXg+vHT2fDN7niHJCJRUp6kMsjd/+ru29x9q7s/CZzn7i8CSYfqaGaDzCzXzBab2dhDtLvYzNzMssLHdc3sGTObY2azzCy7RNvLzGy2mc0zs3sOWs6lZjY/nPd8OV6bxEnTBnV4bEQGG3fsYYwKT4pUG+VJKvvCL+ta4e3SEvNK/SYwswRgHHAu0BMYbmY9I7RLBMYAn5eYfA2Au/cGzgbuC5+7BXAvMNDdjwVamtnAcDldgV8BJ4fzflaO1yZx1CutKXcOOZaPF2/gwXcWxjscEYmC8iSVEcAVQH54uwIYaWYNgNGH6HcCsDi8pv0egs1lQyK0uxO4B9hVYlpP4F0Ad88HNgNZQCdgobsXhO3eAS4K718DjHP3TSX6SSV32fHtuCSzDY/8dzHvLdC/TKSqK0/p+6XufoG7J4e3C9x9sbvvdPePDtE1DVhV4nFeOG0/M+sHtHX3Nw7qOwsYYma1zawjkAm0BRYD3c2sg5nVBoaG0wG6Ad3M7GMz+8zMBkUKysyuNbMcM8spKCiI1EQq2J1De9GjVRN+9uJMVm1U4UmRqqw8R3+1CY/0yjezdWb2spm1KceyI1Uy3r+5zMxqAQ8AN0do9zRBEsoBHgQ+AQrDUcgo4EXgQ2A5UHwWXW2gK5ANDAeeMrNm3wnA/Ul3z3L3rJQUXRW5MqhfJ4HHR2Swb59z/fMqPClSlZVn89czwCSCIpJpwOvhtLLkcWAUAcGRY2tKPE4EegFTzWw5MACYZGZZ7l7o7je6e193HwI0AxYBuPvr7t7f3U8Ecounh8/3mrvvdfdl4byu5YhTKoEOyY34y6V9mJ23hTtenx/vcETkCJUnqaS4+zPhF32huz8LlOcn/pdAVzPraGZ1gWEEyQkAd98Sbk7r4O4dgM+Awe6eY2YNzawRgJmdTTBKmR8+Tg3/JgHXAU+Fi3wVOCOcl0ywOWxpOeKUSuKcY4/h/53WifGfr+SVGXnxDkdEjkB5ksp6MxtpZgnhbSSwoaxO7l5IsCN/MvAVMNHd55nZHWY2uIzuqcB0M/sK+CXBwQHFHjKz+cDHwJ/dvfiwocnAhnDee8DP3b3MOKVy+fk56ZzQsTm/+tcccr9W4UmRqsbKqr9kZu2AR4ETCfaJfAKMcfeVsQ8vtrKysjwnJyfeYchB8rfu4vuPfERivdq8NvpkEuvXiXdIIlKCmU1z96xI88pz9NdKdx/s7inunuruQ4ELox6lSCi1SX0eHd6PFRt3qPCkSBVzpAUlb4pqFCIH6d+pBb84J5235nzN0x8vj3c4IlJOR5pUIh0uLBJV157Wie/1bMldb31FzvKN8Q5HRMrhSJOKtkdIzJkZ917Sh7SkBlz//HTWq/CkSKVXalIppeT9VjPbRnDOikjMNW1Qh8dHZLJ5x17GTFDhSZHKrtSk4u6J7t4kwi3R3WtXZJBSs/Vs3YQ7h/bikyUbuH9KbrzDEZFD0JUfpUq4NKstl2W1Zdx7S3j3q3XxDkdESqGkIlXG74ccS89WTbhRhSdFKi0lFaky6tdJ4ImRmTgwavw0du1V4UmRykZJRaqUdi0acv+lfZm7eiu/V+FJkUpHSUWqnLN7tuQnp3dmwhcreXmaCk+KVCZKKlIl3fK9bgzo1JxbX53Dgq+3xjscEQkpqUiVVDuhFo8Mz6BJ/TqMem46W3ftjXdIIoKSilRhKYn1ePTyDFZu3MEvXlLhSZHKQElFqrQTOjZn7KDu/Gfe1zz14bJ4hyNS4ympSJX341M7MujYY/jzfxbwxTIVnhSJJyUVqfLMjHsuOY62SQ0Y/fx08rftindIIjWWkopUC03q1+HxkZls3RUUniws2hfvkERqJCUVqTZ6tGrCH4b25rOlG7lvysJ4hyNSIympSLVycWYbhp/QlsenLmHKfBWeFKloSipS7fz2gmPpldaEmybOZOUGFZ4UqUhKKlLt1K+TwOMjMjFUeFKkoimpSLXUtnlDHrisL/PWbOV3k+bFOxyRGkNJRaqtgT1acl12Z174chUv5ayKdzgiNYKSilRrN53djZM6t+A3r85l/hoVnhSJNSUVqdZqJ9Ti4eH9aNawDqPGT2PLThWeFIklJRWp9pIb12Pc5Rms3rSTn780S4UnRWJISUVqhKwOzRl7bnfenr+OJz9YGu9wRKotJRWpMa4+pSPn9T6Geybn8vnSDfEOR6RaUlKRGsPMuPui42jfvCGjJ8wgf6sKT4pEm5KK1CiJ9evw2MgMtu3ay2gVnhSJOiUVqXG6H9OEP/2gN18s28i9b+fGOxyRakVJRWqkCzPacHn/dvz1/aW8Pe/reIcjUm3ENKmY2SAzyzWzxWY29hDtLjYzN7Os8HFdM3vGzOaY2Swzyy7R9jIzm21m88zsnrKWJVKa28/vSe+0ptz80ixWbNge73BEqoWYJRUzSwDGAecCPYHhZtYzQrtEYAzweYnJ1wC4e2/gbOA+M6tlZi2Ae4GB7n4s0NLMBpaxLJGI6tdJ4LERGdQy4yfPTVfhSZEoiOVI5QRgsbsvdfc9wAvAkAjt7gTuAUoeitMTeBfA3fOBzUAW0AlY6O4FYbt3gIvKWJZIqYLCk334au1Wbn9tbrzDEanyYplU0oCSVfzywmn7mVk/oK27v3FQ31nAEDOrbWYdgUygLbAY6G5mHcysNjA0nH6oZX2LmV1rZjlmllNQUHCoplJDnNm9JTec2YWJOXlM/FKFJ0WORiyTikWYtr8+hpnVAh4Abo7Q7mmCJJQDPAh8AhS6+yZgFPAi8CGwHCgsY1nfDsD9SXfPcveslJSUw3pBUn397KxunNIlmdtem8vc1VviHY5IzGzfXcjb875m9eadMVl+LJNKHuEoItQGWFPicSLQC5hqZsuBAcAkM8ty90J3v9Hd+7r7EKAZsAjA3V939/7ufiKQG04vdVkxfH1SjSTUMh4a1pekhnW5bvx0FZ6UasPdWbRuG3/7YCkjnvqMfndM4dp/TOPN2WvK7nwEasdkqYEvga7h5qvVwDDg8uKZ7r4FSC5+bGZTgVvcPcfMGgLm7tvN7GyCUcr8sF2qu+ebWRJwHXDpoZYVw9cn1UyLxvUYNyKDy/76KTdPnMWTV2RSq1akAbdI5bZ9dyGfLNnA1Nx8puYW7B+VdGvZmKtO7kB2egpZ7ZvH5LljllTcvdDMRgOTgQTgaXefZ2Z3ADnuPukQ3VOByWa2jyAhXVFi3kNm1ie8f4e7L4xF/FIzZbZP4tfn9eCON+bz1w+WMiq7c7xDEimTu7OkYPv+JPLFso3sKdpHw7oJnNwlmevO6Ex2eippzRrEPBaryWXAs7KyPCdHgxn5Nndn9IQZ/HvOWsb/eAAndm4R75BEvmPHnkI+XbKB98JEkrcpGI10TW1MdnoK2empZHVIol7thKg/t5lNc/eIuxdiuflLpEoqLjz51dqt3DBhBm+NOYXUJvXjHZbUcO7O0vXbeW9BPu8vLODzpQdGIyd1TuYnp3cmOz2FNkkN4xqnkopIBI3r1eaJkZkMefRjRj8/g/HX9KdOgqoaScXauaeIT5eu570FBUxdmM+qjcFopEtqY354Ynuy01M5vmNsRiNHSklFpBTdWiZy14W9+dmLM7l3ci6/Pq9HvEOSas7dWbZ+O1NzC3gvN5/Pl21kT+E+GtRJ4OQuLbj2tM5kd0uhbfP4jkYORUlF5BCG9ksjZ8VGnvxgKRntkhjU65h4hyTVzM49RXy2NDhS673cAlZu3AFA55RGXDGgPdnpKRzfoTn161Se0cihKKmIlOG283syJ28LP39pFunHJNIxuVG8Q5IqLhiNBEnk86Ub2F24j/p1anFy52SuObUj2emplXo0cig6+ktHf0k55G3awfmPfMQxTerzynUn06Bu1fjVKJXDrr1FfLp0A++Hm7VWbAhGI52SG5Gdnkp2egondKw6oxEd/SVylNokNeSBy/ryo2e/5DevzuUvlxyHmU6MlNItD0cjUxcW8OmSA6OREzu14OpTOpLdLZV2LarmaORQlFREyumM9FRuOLMrD7+7iKwOSQw/oV28Q5JKZNfe4n0jBby/sIBl64Nr9HRMbsTl/duRnZ5K/yo0GjlSSioih+GnA7syY+UmfjtpHr3TmtIrrWm8Q5I4WrEhOFJram4+ny7dwK69+6hXuxYndm7BVScF5VDat6hZ++C0T0X7VOQwbdy+h+8//CEJtYw3bziVpg3rxDskqSC79hbx+bKNTM3N5/3cApaGo5EOLRru3zcyoFOLaj8a0T4VkShq3qju/sKTN02cyd9+mKXCk9XYyg07mLowKIXyyZL1+0cjAzq14IrwBEQdEXiAkorIEchol8St5/Xgd6/P5/H3l3D9GV3iHZJEya69RXyxbGOwWWthPksLgtFI+xYNuSyrLdndUxnQsYWOACyFkorIEbrypA5MW7mZ+97OpV+7ZpzUObnsTlIprdq4Y3+F30+WbGDn3iLqhqORkf3bc0Z3jUbKS0lF5AiZGX++sDfz12xhzIQZvHHDqRzTVIUnq4LdhUV8uWxTWOE3nyXhaKRd84ZcktWG7PQUTuyUrNHIEVBSETkKjYoLT477mNHPT2fCtQNUeLKSWrVxB1MXFvB+bj6fLNnAjj3BaKR/x+Zc3j8oh9IpuZHOPzpKSioiR6lrWHjypy/M5O5/L+A35/eMd0hCMBrJWb6J9xYEJyAuzv8GgDZJDbgoIxyNdG5Bw7r6GowmrU2RKBjSN41pKzbx1EfLyGyfxLm9W8U7pBpp9eadQU2tBcGRWjv2FFE3oRb9OzVn2PFtyU5PpXOKRiOxpKQiEiW3fr8Hs/K28PN/zib9mEQ6pTSOd0jV3p7CfeQs38jUhQW8tyCfReFoJK1ZAy7MSCO7Wyondm5Bo3r6qqsoOvlRJz9KFK3evJPzH/6Q1MT6vHL9Sdq0EgNrNu/cf72RTxavZ/ueIuokGP07tggvo5tC55TGGo3EkE5+FKkgac0a8OCwflz1zBf85pW53HdpH325HaU9hfvIWbFxf4XfhesOjEaG9ksjOz2VkzQaqTT0XxCJstO7pfDTgV158J1FZHZIYkT/9vEOqcpZu2Xn/ppaHy/ewDe7C6mTYBzfoTmXZLYlOz2FLqkajVRGSioiMTDmzK5MX7mZ30+aT++0phzXplm8Q6rU9hbtI2f5JqYuDGpqLfh6GwCtm9ZncN/WZHdL4aQuyTTWaKTS0z4V7VORGNm4fQ/nP/whZsabY06hWcO68Q6pUvl6y679Z7F/tHj9/tFIVvvmnNE9hez0VLpqNFIpaZ+KSBw0b1SXx0ZmcskTn3DjizP53yuPr9GFJ/cW7WPaik37N2sVj0ZaNa3PBX1akZ2eyskajVR5+u+JxFDfts247fye3P7aPB6bupjRZ3aNd0gVat3WEqORRevZtruQ2rWMrA5JjD23O2ekp9KtpUYj1YmSikiMXTGgPTnLN3H/lIX0a5fEyV2qb+HJvUX7mL5iE1MXFjA1t4Cv1hDY0BQAAA6vSURBVG4F4Jgm9fn+ccWjkRYk1tc1aKorJRWRGDMz7rqwN/PXbmXMhBm8OaZ6FZ5ct3UX74dl4j9ctJ5tu4LRSGb7JH45qDtndE8hvWWiRiM1hJKKSAUICk9mMPjRj7n++em8UIULTxYW7WPGqs1BTa3cAuaHo5GWTepxXq9WZKencHLXZJpoNFIjKamIVJAuqYncfdFx3DBhBne9tYDbL6g6hSfzt4WjkdwCPlxUwNZdhSSEo5FfDEonu1sqPVppNCJKKiIV6oI+rZm2YhNPfxwUnvz+cZWz8GRh0T5mrtocXm+kgHlrgtFIamI9BvU6Zv+RWk0baDQi36akIlLBfn1eD2blbeYX/5xF91aJdK4khScLtu3m/YVBKZQPF5YYjbRL4ufnpJOdnkLPVk00GpFD0smPOvlR4mDN5p2c/8hHJDeuy6vXnxyXwpNF+5yZqzbtL844d3UwGklJrEd2t+Dkw1O6ajQi3xW3kx/NbBDwEJAAPOXufy6l3cXAS8Dx7p5jZnWBvwJZwD7gp+4+NWx7GXBruMw33f0X4fSbgB8DhUAB8CN3XxHDlydyxFo3a8BDw/ryw6e/4Nf/msMDl/WtkBFAwbbdfFA8Glm0ni0791LLILN9MBo5vVswGqnJJ2nK0YlZUjGzBGAccDaQB3xpZpPcff5B7RKBMcDnJSZfA+Duvc0sFfi3mR0PJAH3ApnuXmBmfzezge7+LjADyHL3HWY2CrgHuCxWr0/kaJ3aNYUbz+rG/VMWktmhOVcMiH7hyWA0spn3c/N5L7eAOau3AJDcuB5n92xJdnoKp3ZJoWlDjUYkOmI5UjkBWOzuSwHM7AVgCDD/oHZ3EiSAW0pM6wm8C+Du+Wa2mWDU4sBCdy8I270DXAS86+7vlej/GTAyui9HJPpGn9GF6Ss3cefr8zkurSl92h594cn13wSjkam5BXywqIDNO4LRSL92SdzyvW5kp6dqNCIxE8ukkgasKvE4D+hfsoGZ9QPauvsbZlYyqcwChoSJqC2QGf79L9DdzDqEyxsKRKrSdzXw70hBmdm1wLUA7dq1O+wXJRJNtWoZD1zal/Mf+Yjrxk/njRtOIanR4RWeLNrnzMrbzNTcAt7PzWf26i24Q3LjugzsHo5GuiaroKVUiFgmlUg/g/YfFWBmtYAHgKsitHsa6AHkACuAT4BCd98Ubtp6kWBfyydAp289qdlIglHN6ZGCcvcngSch2FF/WK9IJAaSGtXlsREZXPLEp9w4cSZPl6Pw5IZvdvPBonA0srCATeFopG/bZtx0VjAaOba1RiNS8WKZVPIIRhfF2gBrSjxOBHoBU8MdlMcAk8xssLvnADcWNzSzT4BFAO7+OvB6OP1aoKhEu7MIduKf7u67Y/CaRGKiT9tm3HZBT257dS6PvreYMQO/XXiyaJ8zOxyNTF1YwOy8zbhDi0Z1OSM9lezuqZzaJfmwRzki0RbLpPIl0NXMOgKrgWHA5cUz3X0LsL+ynplNBW4Jj/5qSHC483YzO5tglDI/bJca7mdJAq4DLg2n9yM4YmyQu+fH8HWJxMTI/u2YtnwjD7yzkH7tmnFs66bhvpF8Pli0no3b92DhaORnA7txRvcUerVuqtGIVCoxSyruXmhmo4HJBIf/Pu3u88zsDiDH3ScdonsqMNnM9hEkpCtKzHvIzPqE9+9w94Xh/XuBxsBL4chnpbsPjuJLEokpM+NPYeHJH/89hz1F+3APrstyercUstNTOK1rikYjUqnp5Eed/CiVzNKCb/jTWwvoldaEM9JT6Z2m0YhULrryo0gV0imlMU9dGfHzKlLpVc3a2yIiUikpqYiISNQoqYiISNQoqYiISNQoqYiISNQoqYiISNQoqYiISNQoqYiISNTU6DPqzayAoArykUgG1kcxnGhRXIdHcR2+yhqb4jo8RxNXe3dPiTSjRieVo2FmOaWVKYgnxXV4FNfhq6yxKa7DE6u4tPlLRESiRklFRESiRknlyD0Z7wBKobgOj+I6fJU1NsV1eGISl/apiIhI1GikIiIiUaOkIiIiUaOkEoGZ3Whm88xsrplNMLP6ZtbRzD43s0Vm9qKZRbymq5n9yswWm1mumZ1TAXGND59rrpk9bWZ1SulbZGYzw9uhLuUcrbieNbNlJZ6zbyl9rwzX6SIzu7IC4vqwRExrzOzVUvrGcn39NIxpnpn9LJzW3MymhOthipklldI3lusrUlz3mtkCM5ttZq+YWbNS+i43sznh+orq5VRLiet3Zra6xP/ovFL6Dgo/H4vNbGwFxPViiZiWm9nMUvpGdX2Fn/18M5tbYlrE95QFHg7XyWwzyyhlmZlhjIvD9uW7/Ki761biBqQBy4AG4eOJwFXh32HhtCeAURH69gRmAfWAjsASICHGcZ0HWHibECmusP03Fby+ngUuLqNvc2Bp+DcpvJ8Uy7gOavMy8MMKXl+9gLlAQ4Irr74DdAXuAcaGbcYCd1fw+iotru8BtcM2d0eKK5y3HEiuwPX1O+CWMvomhJ/BTkDd8LPZM5ZxHdTmPuD2ilhfwGlABjC3xLSI76nwO+Pf4XfGAODzUpb5BXBi2O7fwLnliUUjlchqAw3MrDbBm2YtcCbwz3D+34GhEfoNAV5w993uvgxYDJwQw7jWuPtbHiJ4E7SJ4vMdcVzl7HcOMMXdN7r7JmAKMKgi4jKzRIL/acSRSgz1AD5z9x3uXgi8D/yA4L3z97BNae+vWK6viHG5+9vhY4DPqPj3V2nrqzxOABa7+1J33wO8QLCeYx5X+Kv+UoIfejHn7h8AGw+aXNp7agjwf+HXxmdAMzNrVbJj+LiJu38afrf8H5Hfk9+hpHIQd18N/AVYSZBMtgDTgM0lPlx5BL+ED5YGrCrxuLR2UYnL3d8unh9u9roC+E8pi6hvZjlm9pmZlevNEYW4/hgOrx8ws3oRusdtfRF8Abzr7ltLWURM1hfBr9vTzKyFmTUk+NXYFmjp7mvD2NcCqRH6xmx9HSKukn5E8Is1EgfeNrNpZnZtlGIqK67R4fvr6VI2F8ZzfZ0KrHP3RaX0j9X6Kqm091R51ktaOP1QbSJSUjlI+OYcQrD5qjXQCDg3QtNIx2JH2uYYlWO2I8VlZiNLNHkM+MDdPyxlEe08KMlwOfCgmXWOcVy/AroDxxNsrvllpO4RplXU+hrOoX9FxmR9uftXBJuRphD8AJgFFB6y0wExW19lxWVmt4aPx5eyiJPdPYPgs3K9mZ0W47geBzoDfQl+NNwXoXvc1hdlv79isr7KqTzr5YjXnZLKd50FLHP3AnffC/wLOIlgiFg7bNOGyJt48vj2r5XS2kUzLszst0AKcFNpnd19Tfh3KTAV6BfLuNx9bTi83g08Q+TNgPFaXy3CeN4srXMM1xfu/r/unuHupxFsslgErCveBBH+zY/QNZbrq7S4CA8IOB8YEW4KidS3eH3lA68Qxc2+keJy93XuXuTu+4C/lfJ88VpftYELgRcP0Tdm66uE0t5T5VkveXx7U2e5152SynetBAaYWcNwu+hAYD7wHnBx2OZK4LUIfScBw8ysnpl1JNih+EUM4/rKzH5MsK19ePgB+w4zSyre/GRmycDJ4WuKZVzFb2Yj2BY7N0LfycD3wviSCHYKT45lXOG8S4A33H1XpI4xXl+YWWr4tx3Bl88EgvdO8dFcpb2/Yrm+IsZlZoMIRpmD3X1HKf0ahfuoMLNGYVyR/t/RjKvkPoAflPJ8XwJdLThysy4wjGA9xyyucNZZwAJ3zyulX0zXVwmlvacmAT8MjwIbQLBpeG3JjuHjbWY2IPz8/JDI78nvKs/e/Jp2A34PLCD4R/+D4GiuTgQJYjHwElAvbDsYuKNE31sJjjjJpZxHSxxlXIXh880Mb7eHbbOAp8L7JwFzCIboc4CrKyCu/4bPNRd4Dmh8cFzh4x+F63Qx8D+xjiucPhUYdFDbilxfHxIkqVnAwHBaC+Bdgl+77wLN47C+IsW1mGD7e/H764lwemvgrfB+p7DPLGAecGsFxPWP8H8zm+BLstXBcYWPzwMWhp+RmMcVTn8W+MlBbWO6vggS2lpgL8Eo4+pDvKcMGBeukzlAVonlzDzoMzE3bPcoYQWWsm4q0yIiIlGjzV8iIhI1SioiIhI1SioiIhI1SioiIhI1SioiIhI1SioiIhI1SioilYgFlwy4uOyWIpWTkoqIiESNkopIGcysg5l9ZWZ/s+CCTG+bWQMzm2pmWWGbZDNbHt6/ysxeNbPXLbhQ2Wgzu8nMZoRVj5uX83kzzez9sJLt5BKlb64xsy/NbJaZvRyWomlqwYWfaoVtGprZKjOrY2adzew/4XI+NLPuYZtLLLjI1Cwz+yAmK09qHCUVkfLpCoxz92OBzcBFZbTvRVDh+ATgj8AOd+8HfEpQR+mQLLiUwSMEFzrLBJ4OlwPwL3c/3t37ENQzu9rdtxCU/Tg9bHMBMNmDYppPAjeEy7mFoKI1wO3AOeFyBpcVk0h51C67iYgQVDwuvjTsNKBDGe3fc/dtBEX5tgCvh9PnAMeV4/nSCRLTlKCeHwkEtZ0AepnZH4BmQGMOFJV8EbiMoPjpMOAxM2tMUMvsJTtwNdjia9t8DDxrZhMJqjiLHDUlFZHy2V3ifhHQgKCYZ/Fov/4h2u8r8Xgf5fvcGTDP3U+MMO9ZYKi7zzKzq4DscPok4K5w81omQVHPRgQXmOt78ELc/Sdm1h/4PjDTzPq6+4ZyxCZSKm3+Ejlyywm+vOHAZRGiJRdIMbMTIdgcZmbHhvMSgbXhJrIRxR3c/RuCStoPEZT2L/LgypbLzOyScDlmZn3C+53d/XN3vx1Yz3ev9Chy2JRURI7cX4BRZvYJkBzNBXtwTfWLgbvNbBZB2fmTwtm3AZ8TXHVwwUFdXwRG8u0LRI0Arg6XM48D12m/18zmmNlc4AOCfTIiR0Wl70VEJGo0UhERkajRjnqRODCzcQSXKS7pIXd/Jh7xiESLNn+JiEjUaPOXiIhEjZKKiIhEjZKKiIhEjZKKiIhEzf8HSiR4vYK8qp8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid_search.cv_results_[ 'std_test_score' ]\n",
    "\n",
    "n_leafs = len(num_leaves_s)\n",
    "\n",
    "x_axis = num_leaves_s\n",
    "plt.plot(x_axis, -test_means)\n",
    "#plt.errorbar(x_axis, -test_means, yerr=test_stds,label = ' Test')\n",
    "#plt.errorbar(x_axis, -train_means, yerr=train_stds,label = ' Train')\n",
    "plt.xlabel( 'num_leaves' )\n",
    "plt.ylabel( 'Log Loss' )\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. min_child_samples\n",
    "叶子节点的最小样本数目\n",
    "\n",
    "叶子节点数目：90，共9类，平均每类10个叶子节点\n",
    "每棵树的样本数目数目最少的类（稀有事件）的样本数目：200 * 2/3 * 0.7 = 100\n",
    "所以每个叶子节点约100/10 = 10个样本点\n",
    "\n",
    "搜索范围：10-50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done   8 out of  12 | elapsed:  5.4min remaining:  2.7min\n",
      "[Parallel(n_jobs=4)]: Done  12 out of  12 | elapsed:  7.7min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "             error_score=nan,\n",
       "             estimator=LGBMClassifier(boosting_type='goss', class_weight=None,\n",
       "                                      colsample_bytree=0.7,\n",
       "                                      importance_type='split',\n",
       "                                      learning_rate=0.1, max_bin=127,\n",
       "                                      max_depth=7, min_child_samples=20,\n",
       "                                      min_child_weight=0.001,\n",
       "                                      min_split_gain=0.0, n_estimators=328,\n",
       "                                      n_jobs=4, num_class=9, num_leaves=90,\n",
       "                                      objective='multiclass', random_state=None,\n",
       "                                      reg_alpha=0.0, reg_lambda=0.0,\n",
       "                                      silent=False, subsample=1.0,\n",
       "                                      subsample_for_bin=200000,\n",
       "                                      subsample_freq=0),\n",
       "             iid='deprecated', n_jobs=4,\n",
       "             param_grid={'min_child_samples': range(10, 50, 10)},\n",
       "             pre_dispatch='2*n_jobs', refit=False, return_train_score=False,\n",
       "             scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'goss',\n",
    "          'objective': 'multiclass',\n",
    "          'num_class':9, \n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':n_estimators_1,\n",
    "          'max_depth': 7,\n",
    "          'num_leaves':90,\n",
    "          'max_bin': 127, #2^6,原始特征为整数，很少超过100\n",
    "          'colsample_bytree': 0.7,\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "min_child_samples_s = range(10,50,10) \n",
    "tuned_parameters = dict( min_child_samples = min_child_samples_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=4,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4980377337085618\n",
      "{'min_child_samples': 30}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3hV9Z3v8fc3dy4BEkgQkkC4RCChCpKg4o3RykUFqna8dWZwOj1Onam25+icaluHi6MztdPqzNTpM0o709pzRpBpa0AHD951BE1ALoYAhgAmECEXEi4Jue3f+WPvMJEGshOSrH35vJ5nP2avrLX35+fSfPZea+3fNuccIiISfWK8DiAiIt5QAYiIRCkVgIhIlFIBiIhEKRWAiEiUivM6QE+MGjXKZWdnex1DRCSsbNmypcY5l3b28rAqgOzsbIqLi72OISISVszsYFfLdQhIRCRKqQBERKKUCkBEJEqpAEREopQKQEQkSqkARESilApARCRKRXwB+HyONUUVbPjkc6+jiIiElLD6IFhvOOCFzQc5euI01+SMYkhixA9ZRCQoEf8OIDbGWLEkjyPHm/mnN8u8jiMiEjIivgAALhuXwu2XZfLz98vZV33S6zgiIiEhKgoA4LsLp5AYF8vKdbvQ12CKiERRAaQnJ/GdL+fwzt5qXi896nUcERHPRU0BACydk01O+lBWri/hdGu713FERDwVVQUQHxvD8sV5VNQ18dy75V7HERHxVFQVAMBVk0excPpF/PPbZVQea/Q6joiIZ6KuAAC+f/M0AJ58tdTjJCIi3onKAshMGcxfzJ3Mqzs/57/KaryOIyLiiagsAID7rp1IVuoglhWW0Nru8zqOiMiAi9oCSIqP5bGbcyk7epJffnDA6zgiIgMuagsA4Mbc0Vx3cRr/8PqnVJ9o9jqOiMiAiuoCMDOWLcrldFs7P9yw2+s4IiIDKqgCMLMFZrbHzMrM7JEufn+vmVWb2bbA7RudfrfUzD4N3JZ2Wp5gZs+Z2V4z221mt/fNkHpmYtpQvn71BNZuqWTLwWNeRBAR8US3BWBmscCzwEIgF7jbzHK7WHW1c25G4LYqsG0qsAy4HJgNLDOzlMD63weOOucuDjzuOxc8ml564Poc0pMTWV5YQrtP8wSJSHQI5h3AbKDMOVfunGsBXgSWBPn484GNzrk659wxYCOwIPC7rwN/C+Cc8znnPLsec2hiHN+/eRo7DzWwprjCqxgiIgMqmALIADr/VawMLDvb7Wa2w8zWmlnW+bY1sxGB+4+b2VYze8nMRnf15GZ2n5kVm1lxdXV1EHF7Z/GlY5mdncpTG3ZT39jSb88jIhIqgikA62LZ2cdJ1gHZzrlLgNeBX3azbRyQCfyXc+4yYBPw9109uXPuOedcvnMuPy0tLYi4vWNmLF+cR0NTKz/ZuLffnkdEJFQEUwCVQFan+5nA4c4rOOdqnXMd11E+D8zqZttaoBH4bWD5S8BlPUreD3LHDuNrl4/n15sPsuvwca/jiIj0q2AKoAjIMbMJZpYA3AUUdl7BzMZ0ursY6Jhk5zVgnpmlBE7+zgNec/5vZFkHzA2sdwOwq9ej6EMPzbuY4YPiWV5Yoi+OEZGI1m0BOOfagG/h/2NeCqxxzpWY2UozWxxY7UEzKzGz7cCDwL2BbeuAx/GXSBGwMrAM4LvAcjPbAfwx8FDfDav3RgxO4K/mT+WjA3UUbj/c/QYiImHKwulVbn5+visuLu7352n3OZY8+z7VJ5p586G5DEmM6/fnFBHpL2a2xTmXf/byqP4k8LnExhgrFk/nyPFm/unNMq/jiIj0CxXAOcwan8Ltl2Xy8/fLKa8+6XUcEZE+pwI4j+8unEJiXCwr1u3SCWERiTgqgPNIT07iO1/O4Z291bxeetTrOCIifUoF0I2lc7KZnD6Ux9fv4nRru9dxRET6jAqgG/GxMSxflMdndY08/26513FERPqMCiAIV+eMYuH0i3j27TIO1Td5HUdEpE+oAIL0/ZunAfDEKyHxgWURkQumAghSZspg7r9uMq/u/Jz/KvNs5moRkT6jAuiBP79uIlmpg1heWEJru8/rOCIiF0QF0ANJ8bE8dnMunx49ya82HfQ6jojIBVEB9NCNuaO59uI0ntm4l+oTzd1vICISolQAPWRmLFuUy+m2dn64YbfXcUREek0F0AuT0oby9asnsHZLJVs/O+Z1HBGRXlEB9NID1+eQnpzIspdLaPdpniARCT8qgF4amhjH926axs5DDawpruh+AxGREKMCuABLZoylIDuFH722h4bGVq/jiIj0iArgApgZyxfnUd/Ywk827vE6johIj6gALlDe2OF87fLxvLD5IKVVx72OIyISNBVAH3ho3sUMHxTPspdL9MUxIhI2VAB9YMTgBB6eP4WPDtRRuP2w13FERIKiAugjdxWMY3rGMJ58tZRTzW1exxER6ZYKoI/ExhgrFk/nyPFmfvpWmddxRES6pQLoQ7PGp3DbZRmseq+c8uqTXscRETkvFUAfe2ThVBLjYlm5fpdOCItISFMB9LH05CS+8+Uc3t5TzRulR72OIyJyTiqAfrB0TjaT04eycv0uTre2ex1HRKRLKoB+EB8bw/JFeXxW18jz75Z7HUdEpEsqgH5ydc4oFuRdxLNvl3GovsnrOCIiv0cF0I9+cMs0nIMnXyn1OoqIyO9RAfSjzJTB/MXcybyys4oPymq8jiMi8gUqgH7259dNJDNlEMsKS2ht93kdR0TkDBVAP0uKj+WxW3L59OhJfrXpoNdxRETOUAEMgHm5o7n24jSe2biX6hPNXscREQFUAAPCzFi2KJfTbe08tWG313FERAAVwICZlDaUr181gZe2VPLxZ8e8jiMiElwBmNkCM9tjZmVm9kgXv7/XzKrNbFvg9o1Ov1tqZp8Gbku72LbQzD65sGGEhwduyCE9OZFlhSX4fJonSES81W0BmFks8CywEMgF7jaz3C5WXe2cmxG4rQpsmwosAy4HZgPLzCyl02PfBkTNtJlDE+P43k3T2FHZwJriCq/jiEiUC+YdwGygzDlX7pxrAV4ElgT5+POBjc65OufcMWAjsADAzIYC/wv4m57HDl9LZoylIDuFp17bQ0Njq9dxRCSKBVMAGUDnl6uVgWVnu93MdpjZWjPLCmLbx4EfA43ne3Izu8/Mis2suLq6Ooi4oc3MWL44j/rGFp5+fa/XcUQkigVTANbFsrMPYK8Dsp1zlwCvA78837ZmNgOY7Jz7bXdP7px7zjmX75zLT0tLCyJu6MsbO5x7Lh/HrzYdoLTquNdxRCRKBVMAlUBWp/uZwBe++dw5V+uc67jA/XlgVjfbXgnMMrMDwPvAxWb2dk/Dh7OH501h+KB4lhWW6ItjRMQTwRRAEZBjZhPMLAG4CyjsvIKZjel0dzHQMfvZa8A8M0sJnPydB7zmnPuZc26scy4buBrY65ybe2FDCS8jBifw8PwpfLS/jnU7qryOIyJRqNsCcM61Ad/C/8e8FFjjnCsxs5Vmtjiw2oNmVmJm24EHgXsD29bhP9ZfFLitDCwT4K6CcUzPGMaTr5RyqrnN6zgiEmUsnA4/5Ofnu+LiYq9j9KktB+u4/WebuH/uJL67YKrXcUQkApnZFudc/tnL9Ulgj80an8ptl2Ww6r1y9tec8jqOiEQRFUAIeGThVBLjYlm5rsTrKCISRVQAISA9OYlv35DDW3uqeaP0iNdxRCRKqABCxNI52UxKG8KKdbs43drudRwRiQIqgBCREBfD8sV5fFbXyKr3yr2OIyJRQAUQQq7JSWNB3kU8+9Y+Dtc3eR1HRCKcCiDEfP/mafic44lXS7tfWUTkAqgAQkxW6mDunzuJV3ZU8UFZjddxRCSCqQBC0Devm0RmyiCWryuhtd3ndRwRiVAqgBCUFB/LY7fksvfISV7YdNDrOCISoVQAIWpe7miuyRnF0xv3Un2iufsNRER6SAUQosyMZYvyaGpt56kNu72OIyIRSAUQwianD+XPrp7AS1sq+fizY17HEZEIowIIcQ/ckEN6ciLLC0vw+cJn5lYRCX0qgBA3NDGOR2+ayvbKBl7aUtH9BiIiQVIBhIGvzMggf3wKP9ywh4bGVq/jiEiEUAGEATNjxZI86htbePr1vV7HEZEIoQIIE3ljh3PP5eN4YfNBdn9+3Os4IhIBVABh5KEbp5CcFMeyl0sIp6/yFJHQpAIIIylDEnh43hQ+3F/Huh1VXscRkTCnAggzd88eR97YYTz5Simnmtu8jiMiYUwFEGZiY4yVS/L4/Phpnn2rzOs4IhLGVABhaNb4VG6bmcGq9/azv+aU13FEJEypAMLUIwunkhAXw8p1JV5HEZEwpQIIU+nDkvj2DTm8taeaN0qPeB1HRMKQCiCMLZ2TzaS0Iaxcv4vTre1exxGRMKMCCGMJcTEsX5zHwdpGfv7+fq/jiEiYUQGEuWty0pifN5qfvlnG4fomr+OISBhRAUSAH9yci885nni11OsoIhJGVAARICt1MPfPncQrO6r4YF+N13FEJEyoACLEN6+bRGbKIFYU7qK13ed1HBEJAyqACJEUH8sPbs5lz5ETvLDpoNdxRCQMqAAiyPy80VyTM4qnX99Lzclmr+OISIhTAUQQM2PZojyaWtp5asNur+OISIhTAUSYyelD+frVE1hTXMm2inqv44hICFMBRKAHrp9MWnIiy17+BJ9PXxwjIl1TAUSg5KR4vnfTVLZXNvDSlgqv44hIiAqqAMxsgZntMbMyM3uki9/fa2bVZrYtcPtGp98tNbNPA7elgWWDzewVM9ttZiVm9nd9NyQB+MqMDPLHp/DUhj00NLV6HUdEQlC3BWBmscCzwEIgF7jbzHK7WHW1c25G4LYqsG0qsAy4HJgNLDOzlMD6f++cmwrMBK4ys4UXPhzpYGYsX5xHXWMLT2/c63UcEQlBwbwDmA2UOefKnXMtwIvAkiAffz6w0TlX55w7BmwEFjjnGp1zbwEEHnMrkNnz+HI+0zOGc8/scbyw+SC7Pz/udRwRCTHBFEAG0PlAcmVg2dluN7MdZrbWzLKC3dbMRgCLgDe6enIzu8/Mis2suLq6Ooi40tnD86aQnBTHspdLcE4nhEXkvwVTANbFsrP/kqwDsp1zlwCvA78MZlsziwP+HfhH51x5V0/unHvOOZfvnMtPS0sLIq50ljIkgYfnTeHD/XWs31HldRwRCSHBFEAlkNXpfiZwuPMKzrla51zHR0+fB2YFue1zwKfOuWd6Elp65u7Z48gbO4wnXy3lVHOb13FEJEQEUwBFQI6ZTTCzBOAuoLDzCmY2ptPdxUDHvMSvAfPMLCVw8ndeYBlm9jfAcOA7FzYE6U5sjLFicR5VDad59q0yr+OISIjotgCcc23At/D/4S4F1jjnSsxspZktDqz2YOByzu3Ag8C9gW3rgMfxl0gRsNI5V2dmmcD38V9VtPXsS0el7+Vnp3LbzAxWvbefAzWnvI4jIiHAwunEYH5+visuLvY6Rtg6evw01//4HWZPSOUX9xZ4HUdEBoiZbXHO5Z+9XJ8EjiLpw5J48IbJvLn7KG/uPuJ1HBHxmAogytw7ZwIT04awYt0uTre2ex1HRDykAogyCXExLF+Ux8HaRn7+/n6v44iIh1QAUejai9OYnzean75ZxuH6Jq/jiIhHVABR6gc35+JzjidfLe1+ZRGJSCqAKJWVOphvXjeJ9Tuq2LSv1us4IuIBFUAUu3/uJDJTBrG8sIS2dp/XcURkgKkAolhSfCw/uDmXPUdO8MLmg17HEZEBpgKIcvPzRnNNzih+snEvNSebu99ARCKGCiDKmRnLFuXR1NLOjzbs8TqOiAwgFYAwOX0oX796AquLK9hWUe91HBEZICoAAeCB6yeTlpzIspc/wecLn/mhRKT3VAACQHJSPI8unMr2ygbWbqn0Oo6IDAAVgJxx68wMZo1P4YcbdtPQ1Op1HBHpZyoAOcPM/8UxdY0tPL1xr9dxRKSfqQDkC6ZnDOee2eN4YfNB9nx+wus4ItKPVADyex6eN4XkpDiWFX5COH1hkIj0jApAfk/KkAQemjeFzeV1vLKzyus4ItJPVADSpXtmjyN3zDCeeKWUxpY2r+OISD9QAUiXYmOMlUvyqGo4zbNvlXkdR0T6gQpAzik/O5VbZ2bw/Lv7OVBzyus4ItLHVAByXo8unEp8rPH4+l1eRxGRPqYCkPNKH5bEt7+cwxu7j/Lm7iNexxGRPqQCkG7dO2cCE9OGsHLdLprb2r2OIyJ9RAUg3UqIi2H5ojwO1Day6r39XscRkT6iApCgXHtxGvNyR/PTN8uoamjyOo6I9AEVgATtsVty8TnHE6+Ueh1FRPqACkCClpU6mG9eN4n1O6rYtK/W6zgicoFUANIj98+dRMaIQaxYV0Jbu8/rOCJyAVQA0iNJ8bE8dss0dn9+gl9vPuh1HBG5ACoA6bH5eRdxTc4ofrxxLzUnm72OIyK9pAKQHjMzli3Ko6mlnR9t2ON1HBHpJRWA9Mrk9KH86VXZrNlSwfaKeq/jiEgvqACk1x68IYdRQxP568ISfD59cYxIuFEBSK8lJ8Xz6MKpbK+oZ+2WSq/jiEgPqQDkgtw6M4NZ41P44YbdNDS1eh1HRHogqAIwswVmtsfMyszskS5+f6+ZVZvZtsDtG51+t9TMPg3clnZaPsvMdgYe8x/NzPpmSDKQzIwVi/Ooa2zhmdf3eh1HRHqg2wIws1jgWWAhkAvcbWa5Xay62jk3I3BbFdg2FVgGXA7MBpaZWUpg/Z8B9wE5gduCCx2MeGN6xnDumT2OX206yJ7PT3gdR0SCFMw7gNlAmXOu3DnXArwILAny8ecDG51zdc65Y8BGYIGZjQGGOec2Oecc8CvgK73ILyHi4XlTSE6KY1nhJ/h3qYiEumAKIAOo6HS/MrDsbLeb2Q4zW2tmWd1smxH4ubvHxMzuM7NiMyuurq4OIq54IWVIAg/Nm8Lm8jpe2VnldRwRCUIwBdDVsfmzX+KtA7Kdc5cArwO/7GbbYB7Tv9C555xz+c65/LS0tCDiilfumT2O3DHDeOKVUhpb2ryOIyLdCKYAKoGsTvczgcOdV3DO1TrnOuYEeB6Y1c22lYGfz/mYEn5iY4yVS/KoajjNP7+1z+s4ItKNYAqgCMgxswlmlgDcBRR2XiFwTL/DYqBjwvjXgHlmlhI4+TsPeM05VwWcMLMrAlf//Anw8gWORUJAfnYqt87M4Ll3yzlQc8rrOCJyHt0WgHOuDfgW/j/mpcAa51yJma00s8WB1R40sxIz2w48CNwb2LYOeBx/iRQBKwPLAO4HVgFlwD7gP/tsVOKpRxdOJT7WeHz9Lq+jiIS9ppZ2Piir6ZfHtnC6YiM/P98VFxd7HUOC8C/v7ONv/3M3/3pvAX8wNd3rOCJhxTnHzkMNrC6qoHDbYRpb29n06PWkJyf16vHMbItzLv/s5XEXnFSkC3961QRWF1ewYl0JcyaPJDEu1utIIiGvvrGF3318iNXFlZRWHScpPoabpo/hjoIs0oYm9vnzqQCkXyTExbB8UR5/8ouPWPXefv7yDyZ7HUkkJPl8jk3ltawuqmBDyee0tPn4UsZwHv/KdBZfOpbhg+L77blVANJvrr04jXm5o/npm2XcdlkGY4YP8jqSSMioamhibXEla7ZUUFHXxLCkOO4uyOKOgizyxg4fkAwqAOlXj92Sy5d/8g5Pvrqbf7p7ptdxRDzV0ubjzd1HWF1UwTt7q/E5mDNpJA/Pm8L8vItIih/YQ6UqAOlXWamD+eZ1k/iHNz7la5eP44qJI72OJDLgyo6eZE1xBf+xpZLaUy2MHpbIX8ydzB35WYwbOdizXCoA6Xf3z53E2i2VLC8sYf0DVxMXq1nIJfKdam7jlZ1VrCmqoPjgMeJijBumpXNnQRbX5qSFxP8HKgDpd0nxsTx2yzS++eut/HrzQe69aoLXkUT6hXOObRX1rCn2X755qqWdiWlDeHThVG67LJO05L6/kudCqABkQMzPu4irJ4/iJxv3sujSsYzsh0vaRLxSd6qF3358iDVFFew5coJB8bHcfMkY7izIIn98CqH6dScqABkQZsbyxbkseOY9fvTaHv7u9ku8jiRyQXw+x/tlNawurmBjyRFa2n1cmjWCJ2/9EosuHUNyUv9dvtlXVAAyYCanJ/OnV2Wz6v393D17HJdmjfA6kkiPHapv4qXiCl4qruRQfRMjBsfztSvGcWdBFlMvGuZ1vB5RAciAevCGHH637TB/XVjCb++fQ0xMaL41Fumsua2d13cdZXVxBe99Wo1zcE3OKB5ZOJV5eaPD9pPuKgAZUMlJ8TyyYCoPvbSdtVsruSM/q/uNRDyy98gJVhdV8JutlRxrbGXM8CQeuD6HP5yVSVaqd5dv9hUVgAy4W2dm8H8/+ownXy1lf80p5kwaSf74VAYlhOerKIksJ5vbWL/9MKuLK/j4s3riY40bc0dzR34W1+SkERtB71o1G6h4ouzoCR79zU4+/qyeNp8jPtaYmZXCFZNGMmfSSGaOGxG2b6sl/Djn2PrZMVYXVbB+RxWNLe3kpA/lzoIsbp2ZEfZXrZ1rNlAVgHjqVHMbxQeP8cG+Gjbvq2XnoQZ8DhLjYsjPTuHKiSO5ctIoLskcTnwIfHBGIkvNyWZ+u/UQq4srKDt6ksEJsSy6ZCx3FGRx2bgRIXv5Zk+pACQsNDS1UrS/jg/21bKpvJbSquMADEmIpWBCKldOHMmcSaPIHTssot6Ky8Bp9zne/bSaNUUVbNx1hDaf47JxI7izIIubLxnL0MTIOzKuApCwVHeqhQ/La88UQtnRkwAMS4rj8okjA+8QRjJldLKuKJLzqqhr9F++uaWSqobTpA5J4LaZGdxZkEXO6GSv4/UrFYBEhKPHT7OpvJZNgUI4WNsIQOqQBK6cOPLMOYSJo4ZEzNt36b3Tre38v11HWFNUwftlNZjBNTlp3FWQxZenjSYhLjoOK6oAJCIdqm9i075aPthXw6Z9tVQ1nAYgPTmROZP87w7mTBoVEZfsSfBKq46zuqiC3207RH1jKxkjBnFHfhZfzc8kY0T0fS+FCkAinnOOg7WNbOo4ZLSvlpqTzQBkjBgUKAN/KejLaSLPidOtFG4/zJqiCrZXNpAQG8O8vNHcWZDFVZNGRfUhQhWARB3nHGVHT/oLoayWzftrqW9sBWDCqCFcMdFfCFdMHBlyszRKcJxzFB3wX7756s4qmlrbmTI6+czlmylDEryOGBJUABL1fD5H6efH/ecP9tXy0f46TjS3AXDx6KFnLjm9YmIqIwbrD0coO3riNL/Z6p99s7zmFEMT41h06VjuLMji0szhOv9zFhWAyFna2n18cvj4mXMIxQeO0dTajhnkjhnmv+R08kgKslPDYmbHSNfW7uOdvdWsLqrgjd1Hafc58senBC7fHMPghMi7fLOvqABEutHS5mN7Zf2ZQtj6WT0tbT5iY4wvZQw/cw5B01YMrIO1p1hTXMHaLZUcOd7MqKEJ3H5ZJn+Yn8Xk9KFexwsLKgCRHjrd2s7Wg8fOnFTeXqFpKwbK6dZ2NnzyOauLKthUXkuMwdwp6dyRn8UN09L1qfAeUgGIXKBTzW0UHag78zmETzRtRZ/75FADa4or+N3Hhzh+uo2s1EHcmZ/F7bMydeXWBVABiPSxhqZWPtpfd+aQ0e7PTwCatqKnGppaKdzmn4/nk0PHSYiLYeH0i7gzP4srJo6M6ss3+4oKQKSf1Z1qYXPg3cEH+2rYV30K0LQVXXHOsbm8jjXF/ss3m9t8TBszjLsKslgyY6yuwupj5yoAnTYX6SOpQxK46UtjuOlLY4AvTlvxwb5aNu46cma9KyamcuWkUVw5cSST0qJn2oojx0+zdkslLxVXcKC2keTEOL46K5O7CsYxPWNY1Px7CBV6ByAyQCqPNZ6Zw+jsaSs6rjCKxGkrWtt9vLX7KGuKK3hrTzXtPsfsCancVZDFwuljdEXVANAhIJEQ0jFtxQdnCqGGmpMtQORMW7G/5hSriyr4j62VVJ9oJi05ka/OyuSO/CwmjBridbyoogIQCWEd01Z0zGEUrtNWNLW08+rOKlYXV/DR/jpiY4w/mJLGnQXjmDslTVdHeUQFIBJGzp624sP9dZwM0WkrnHPsPNTA6qIKCrcd5kRzG9kjB3NHQRa3X5bJ6GFJnuYTFYBIWOuYtqJj2uuiA3WcbvV5Om1FfWMLv/v4EKuLKymtOk5iXAw3fWkMdxZkcfmEVJ3QDSEqAJEI0jFtxQdltWwqr2HrwXpa2vt/2gqfz7G5vJYXiyrYUPI5LW0+pmcM486CcSy+dCzDB2nOpFCkAhCJYB3TVnScVO7raSuqGppYW1zJmi0VVNQ1MSwpjq/MzOCO/CymZwzvhxFJX1IBiESRM9NWBAqh62krRnJJ5ohznphtbffxRukRVhdV8M7eanwOrpw4krtmZzE/7yKS4nX5Zri4oAIwswXAPwCxwCrn3N+dY72vAi8BBc65YjNLAP4FyAd8wLedc28H1r0b+B7ggMPAHznnas6XQwUg0jsd01Z0nEPomLZicEIss8+atmJ/jX/2zd9sraTmZAujh/335ZvjR+ryzXDU608Cm1ks8CxwI1AJFJlZoXNu11nrJQMPAh92Wvw/AJxzXzKzdOA/zawAiMFfKLnOuRozewr4FrC8N4MTkfMbPiieG3NHc2PuaABqTzbzYadCeHtPNeCfx+hUSztxMcb1U9O5a3YW1+akEafLNyNSMFNBzAbKnHPlAGb2IrAE2HXWeo8DTwEPd1qWC7wB4Jw7amb1+N8NfAwYMMTMaoFhQNkFjENEemDk0MQvTFtx5PhpNpf7vyUtK3Uwt12WQXqyLt+MdMEUQAZQ0el+JXB55xXMbCaQ5Zxbb2adC2A7sCRQGlnArMB6H5nZ/cBO4BTwKfCXXT25md0H3Acwbty4oAYlIj0zelgSS2ZksGRGhtdRZAAF876uq4t5z5w4MLMY4GngoS7W+wX+wigGngE+ANrMLB64H5gJjAV2AI929eTOueecc/nOufy0tLQg4oqISDCCeQdQif/Ve4dM/CdtOyQD04G3Ax/8uAgoNLPFzrli4H92rGhmH+B/tbtybYMAAASkSURBVD8DwDm3L7B8DfBI74chIiI9Fcw7gCIgx8wmBK7quQso7Pilc67BOTfKOZftnMsGNgOLA1cBDTazIQBmdiPQFjh5fAjINbOOl/Q3AqV9NywREelOt+8AnHNtZvYt4DX8l4H+wjlXYmYrgWLnXOF5Nk8HXjMzH/4/+n8ceMzDZrYCeNfMWoGDwL0XNhQREekJfRBMRCTCnetzALq4V0QkSqkARESilApARCRKhdU5ADOrxn/CuDdGAeedayiMRMpYImUcoLGEqkgZy4WOY7xz7vc+SBVWBXAhzKy4q5Mg4ShSxhIp4wCNJVRFylj6axw6BCQiEqVUACIiUSqaCuA5rwP0oUgZS6SMAzSWUBUpY+mXcUTNOQAREfmiaHoHICIinagARESiVEQWgJn9wsyOmtknnZalmtlGM/s08M8ULzMG4xzjWG5mh8xsW+B2k5cZg2VmWWb2lpmVmlmJmX07sDys9st5xhF2+8XMkszsIzPbHhjLisDyCWb2YWCfrA7MAhzSzjOWfzOz/Z32ywyvswbDzGLN7GMzWx+43y/7JCILAPg3YMFZyx4B3nDO5eD/mspw+P6Bf+P3xwHwtHNuRuD26gBn6q024CHn3DTgCuAvzSyX8Nsv5xoHhN9+aQaud85div87OhaY2RXAD/GPJQc4BvyZhxmDda6xAPxVp/2yzbuIPfJtvjhFfr/sk4gsAOfcu0DdWYuXAL8M/PxL4CsDGqoXzjGOsOScq3LObQ38fAL/f9wZhNl+Oc84wo7zOxm4Gx+4OeB6YG1gecjvEzjvWMKOmWUCNwOrAveNftonEVkA5zDaOVcF/v+J8X9XQbj6lpntCBwiCulDJl0xs2z8Xwf6IWG8X84aB4ThfgkcatgGHAU2AvuAeudcW2CVSsKk4M4ei3OuY788EdgvT5tZoocRg/UM8L8BX+D+SPppn0RTAUSKnwGT8L/NrQJ+7G2cnjGzocB/AN9xzh33Ok9vdTGOsNwvzrl259wM/F/1OhuY1tVqA5uqd84ei5lNx/9d41OBAiAV+K6HEbtlZrcAR51zWzov7mLVPtkn0VQAR8xsDEDgn0c9ztMrzrkjgf/QfcDz+P+nDQtmFo//j+b/cc79JrA47PZLV+MI5/0C4JyrB97Gf15jhJl1fFvg2d8BHvI6jWVB4JCdc841A/9K6O+Xq4DFZnYAeBH/oZ9n6Kd9Ek0FUAgsDfy8FHjZwyy91vHHMuBW4JNzrRtKAscxfw6UOud+0ulXYbVfzjWOcNwvZpZmZiMCPw8Cvoz/nMZbwFcDq4X8PoFzjmV3pxcXhv+4eUjvF+fco865zMD3q98FvOmc+xr9tE8i8pPAZvbvwFz8U6geAZYBvwPWAOOAz4A/dM6F9AnWc4xjLv7DDA44APx5xzH0UGZmVwPvATv572Ob38N//Dxs9st5xnE3YbZfzOwS/CcUY/G/GFzjnFtpZhPxv/pMBT4G/ijwCjpknWcsbwJp+A+jbAO+2elkcUgzs7nAw865W/prn0RkAYiISPei6RCQiIh0ogIQEYlSKgARkSilAhARiVIqABGRKKUCEBGJUioAEZEo9f8BehfmqFAdZ7wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid_search.cv_results_[ 'std_test_score' ]\n",
    "\n",
    "x_axis = min_child_samples_s\n",
    "\n",
    "plt.plot(x_axis, -test_means)\n",
    "#plt.errorbar(x_axis, -test_scores, yerr=test_stds ,label = ' Test')\n",
    "#plt.errorbar(x_axis, -train_scores, yerr=train_stds,label =  +' Train')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.50672123, -0.49917791, -0.49803773, -0.49976807])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_means"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### min_child_samples=30"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 列采样参数 sub_feature/feature_fraction/colsample_bytree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 5 candidates, totalling 15 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  12 out of  15 | elapsed:  7.5min remaining:  1.9min\n",
      "[Parallel(n_jobs=4)]: Done  15 out of  15 | elapsed: 34.9min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "             error_score=nan,\n",
       "             estimator=LGBMClassifier(boosting_type='goss', class_weight=None,\n",
       "                                      colsample_bytree=1.0,\n",
       "                                      importance_type='split',\n",
       "                                      learning_rate=0.1, max_bin=127,\n",
       "                                      max_depth=7, min_child_samples=30,\n",
       "                                      min_child_weight=0.001,\n",
       "                                      min_split_gain=0.0, n_estimators=328,\n",
       "                                      n_jobs=4, num_class=9, num_leaves=90,\n",
       "                                      objective='multiclass', random_state=None,\n",
       "                                      reg_alpha=0.0, reg_lambda=0.0,\n",
       "                                      silent=False, subsample=1.0,\n",
       "                                      subsample_for_bin=200000,\n",
       "                                      subsample_freq=0),\n",
       "             iid='deprecated', n_jobs=4,\n",
       "             param_grid={'colsample_bytree': [0.5, 0.6, 0.7, 0.8, 0.9]},\n",
       "             pre_dispatch='2*n_jobs', refit=False, return_train_score=False,\n",
       "             scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'goss',\n",
    "          'objective': 'multiclass',\n",
    "          'num_class':9, \n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.1,\n",
    "          'n_estimators':n_estimators_1,\n",
    "          'max_depth': 7,\n",
    "          'num_leaves':90,\n",
    "          'min_child_samples':30,\n",
    "          'max_bin': 127, #2^6,原始特征为整数，很少超过100\n",
    "          #'colsample_bytree': 0.7,\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "colsample_bytree_s = [i/10.0 for i in range(5,10)]\n",
    "tuned_parameters = dict( colsample_bytree = colsample_bytree_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=4,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , y_train)\n",
    "#grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.49687566033929115\n",
      "{'colsample_bytree': 0.8}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD4CAYAAAAHHSreAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU9dX48c9JQtgMaxIgCSFsgmEPQ0DBuqJoNSAgW7Xaam1Vah9rn1btr09dutpabatP+7hQaatCWGSxKhUqVUQgk0BYBSJbJmEJW9iXJOf3x9zQMRnIBJLcSXLer9e8crfvvedeQs5873KuqCrGGGNMoAi3AzDGGBN+LDkYY4ypxJKDMcaYSiw5GGOMqcSSgzHGmEqi3A6gJsTGxmpKSorbYRhjTL2SnZ29X1Xjgs1rEMkhJSUFr9frdhjGGFOviMjO882z00rGGGMqseRgjDGmEksOxhhjKrHkYIwxphJLDsYYYyqx5GCMMaYSSw7GGGMqseRgjDH1UFmZ8vN/bGRDYXGtrN+SgzHG1EMrtx/k1U+2s2Xv0VpZvyUHY4yph2Z584lpFsUtfTvVyvotORhjTD1z5NRZ3lu/m4wBCTRrElkr27DkYIwx9czC3EJOnS1j4pDOtbYNSw7GGFPPZGbl07tjDP0SW9faNiw5GGNMPfL5niPk+oqZ4OmMiNTadkJKDiIySkQ2i0ieiDweZP69IlIkImucz/0B8+4Rka3O556A6YNFZJ2zzj+Is5ci8pSIFASs69aa2FFjjGkIMrN8NIkUxgxKrNXtVPk+BxGJBF4GRgI+IEtEFqjqxgqLzlTVqRXatgN+CngABbKdtoeAPwEPACuA94BRwPtO0xdU9bcXv1vGGNPwnCkp453VPm5K7Ui7ltG1uq1Qeg7pQJ6qblPVM8AMYHSI678Z+FBVDzoJ4UNglIh0Alqp6meqqsBfgTEXEb8xxjQaizft5dCJs9zpSar1bYWSHBKB/IBxnzOtonEislZEZotI+SX087VNdIbPt86pzrqmiUjbYEGJyAMi4hURb1FRUQi7YYwx9VumN59OrZtxdc+gb/asUaEkh2BXPLTC+EIgRVX7A4uB6VW0vdA6/wR0BwYCu4HngwWlqq+oqkdVPXFxtX+gjDHGTbuLT/LxliLGD04iMqL2LkSXCyU5+IDAm2mTgMLABVT1gKqedkZfBQZX0dbnDFdap6ruVdVSVS1z1pUe2q4YY0zDNdvro0zhzsG192xDoFCSQxbQU0S6ikg0MAlYELiAcw2hXAawyRleBNwkIm2d00M3AYtUdTdwVESGOXcpfR2YH2RddwDrL2K/jDGmwSgrU2Zl+7iyW3uS27eok21WebeSqpaIyFT8f+gjgWmqukFEngG8qroAeEREMoAS4CBwr9P2oIg8iz/BADyjqged4QeBN4Dm+O9SKr9T6TkRGYj/NNMO4NuXupPGGFOfrdh+gF0HT/D9kZfX2TbFf7NQ/ebxeNTr9bodhjHG1IpHZ65h8aa9ZP34xhqtpSQi2arqCTbPnpA2xpgwVnzyLO+t283ogbVXZC8YSw7GGBPGFuYWcrqkjIme5DrdriUHY4wJY5lef5G9vomt6nS7lhyMMSZMbdp9hLW+YiYOqd0ie8FYcjDGmDCV6c0nOjKCMQNrt8heMJYcjDEmDJ0uKWXe6gJG9ulA21ousheMJQdjjAlDizfu49CJs0zw1M0T0RVZcjDGmDA005tPQutmjOgR68r2LTkYY0yYKTx8kk+21l2RvWAsORhjTJiZne1DFe506ZQSWHIwxpiw4i+yl89V3dvTuV3dFNkLxpKDMcaEkRXbDpB/8CQTh7jXawBLDsYYE1Yyvfm0ahbFzX06uhqHJQdjjAkTxSfP8v76PYwemFinRfaCseRgjDFhYkF5kT2XTymBJQdjjAkbmVn5XNGpFX0S6rbIXjCWHIwxJgxsLDzCuoJiJnqS6rzIXjAhJQcRGSUim0UkT0QeDzL/XhEpEpE1zuf+gHn3iMhW53NPwPTBIrLOWecfnHdJIyLtRORDZ/kPnXdPG2NMg1ZeZG+0C0X2gqkyOYhIJPAycAuQCkwWkdQgi85U1YHO5zWnbTvgp8BQIB34acAf+z8BDwA9nc8oZ/rjwBJV7QksccaNMabBOl1Syrw1BdzkUpG9YELpOaQDeaq6TVXPADOA0SGu/2bgQ1U9qKqHgA+BUSLSCWilqp+p/yXWfwXGOG1GA9Od4ekB040xpkH6cONeDrtYZC+YUJJDIpAfMO5zplU0TkTWishsESnfw/O1TXSGg62zg6ruBnB+xgcLSkQeEBGviHiLiopC2A1jjAlPM7PySWzTnOEuFdkLJpTkEOzKiFYYXwikqGp/YDH/+eZ/vrahrPOCVPUVVfWoqicuLq46TY0xJmwUHD7Jsrz9jHOxyF4woSQHHxDY10kCCgMXUNUDqnraGX0VGFxFW58zHGyde53TTjg/94UQozHG1Euzvf6TKHcOTqpiyboVSnLIAnqKSFcRiQYmAQsCFyj/Y+7IADY5w4uAm0SkrXMh+iZgkXO66KiIDHPuUvo6MN9pswAov6vpnoDpxhjToJQX2RvePdbVInvBRFW1gKqWiMhU/H/oI4FpqrpBRJ4BvKq6AHhERDKAEuAgcK/T9qCIPIs/wQA8o6oHneEHgTeA5sD7zgfgV0CmiNwH7ALuvOS9NMaYMPTZtgP4Dp3kv2/u5XYolYj/ZqH6zePxqNfrdTsMY4yplu/NWM1Hn+9j1Y9vdKWWkohkq6on2Dx7QtoYY1xQfMJfZG/MIPeL7AVjycEYY1ywILeAMyVlYfVsQyBLDsYY44KZ3nxSO7Wib2Jrt0MJypKDMcbUsQ2FxawvOBIWpbnPx5KDMcbUsVleH9FREYwemOB2KOdlycEYY+rQqbOlvLO6gJv7dKRNi/AosheMJQdjjKlDH27cS/HJs0zwhNcT0RVZcjDGmDqU6XWK7HUPnyJ7wVhyMMaYOuI7dIJlefsZPziJiDAqsheMJQdjjKkjs7OdInthfkoJLDkYY0ydKCtTZnl9jOgRS1Lb8CqyF4wlB2OMqQPLvzhAweGT3BmmT0RXZMnBGGPqQKY3n9bNm3BTage3QwmJJQdjjKllxSfO8sGGPYwZmBCWRfaCseRgjDG1bH55kb0wLpdRkSUHY4ypZTOz8umT0Io+CeFZZC8YSw7GGFOL1hcUs6EwvIvsBRNSchCRUSKyWUTyROTxCyw3XkRURDzOeLSI/EVE1olIrohcG7DsRBFZKyIbROS5gOn3ikiRiKxxPvdfwv4ZY4yrZnnz/UX2BiS6HUq1VJkcRCQSeBm4BUgFJotIapDlYoBHgJUBk78FoKr9gJHA8yISISLtgd8AN6hqH6CDiNwQ0G6mqg50Pq9d5L4ZY4yrTp0tZd6aQkb16UjrFk3cDqdaQuk5pAN5qrpNVc8AM4DRQZZ7FngOOBUwLRVYAqCq+4DDgAfoBmxR1SJnucXAuIvaA2OMCVP/PFdkr36dUoLQkkMikB8w7nOmnSMig4DOqvpuhba5wGgRiRKRrsBgoDOQB/QWkRQRiQLGONPLjXNOOc0WkaBHVUQeEBGviHiLioqCLWKMMa7KzPIX2buqe3u3Q6m2UJJDsOpQem6mSATwAvBYkOWm4U8mXuBFYDlQoqqHgAeBmcAnwA6gxGmzEEhR1f74exTTgwWlqq+oqkdVPXFxcSHshjHG1J38gyf49Iv93OkJ/yJ7wUSFsIyPL3+rTwIKA8ZjgL7AUhEB6AgsEJEMVfUCj5YvKCLLga0AqroQfyJARB4ASp3pBwLW/Srw6+rtkjHGuO8/Rfbq3yklCK3nkAX0FJGuIhINTAIWlM9U1WJVjVXVFFVNAVYAGarqFZEWItISQERG4u81bHTG452fbYGHgNec8U4B284ANl3qThpjTF0qK1NmZ/uL7CW2ae52OBelyp6DqpaIyFRgERAJTFPVDSLyDOBV1QUXaB4PLBKRMqAAuDtg3u9FZIAz/IyqbnGGHxGRDPynmQ4C91Zrj4wxxmWffrGfgsMnefyW3m6HctFEVateKsx5PB71er1uh2GMMQB89+3VfLK1iJVP3kDTqPCtpSQi2arqCTbPnpA2xpgadPjEGRZt2MOYgYlhnRiqYsnBGGNq0Pw1hf4ie/X0QnQ5Sw7GGFODZmbl0zexFakJrdwO5ZJYcjDGmBqyvqCYjbuPMLGe9xrAkoMxxtSYTKfIXkY9K7IXjCUHY4ypAafOljJvdQG39K1/RfaCseRgjDE1YNGGPRw5VVLvL0SXs+RgjDE1INObT+d2zbmyW/0rsheMJQdjjLlE+QdP8GneAe4c3LleFtkLxpKDMcZcolnZPkRg3OAkt0OpMY06OZw8U8qps6Vuh2GMqcdKy5TZ3nyu7hlXb4vsBdOok8Os7HyG/GwxP5q9llXbD9IQ6kwZY+rWp3n7KSw+xQRPw+k1QGjvc2iwBiS1YWSfDixcW8hM52LS2EFJjEtLIrl9C7fDM8bUA5nefNq0aMLI1A5uh1KjrCorcPx0CR+s38OcHB+fbTuAKqSntGNsWiK39u9Eq2b1/55lY0zNO3T8DEN/sYQpQ5N5KqOP2+FU24WqsjbqnkO5lk2jGDc4iXGDkyg4fJJ5qwuYk+3j8bnr+OmCDdzUpyPj0hK5umcckQ3kTgRjzKWbv6aAM6X1v8heMJYcKkhs05yHr+vBQ9d2Z03+Yebk+FiYu5uFuYXExzTljkGJjE1LolfHGLdDNca4SFWZ6fXRL7F1vS+yF4wlh/MQEQYlt2VQclt+clsq/9q0jzk5Pl5ftp3/+3gbfRNbMS4tiYwBCbS/rKnb4Rpj6tiGwiNs2n2EZ8f0dTuUWhHS3UoiMkpENotInog8foHlxouIiojHGY8Wkb+IyDoRyRWRawOWnSgia0Vkg4g8FzC9qYjMdLa1UkRSLnrvakjTqEhu6deJ1+4Zwoonb+B/bktFFZ5euJGhv1jC/dO9vL9uN6dL7LZYYxqLmVn5NI2KIGNAgtuh1Ioqew4iEgm8DIwEfECWiCxQ1Y0VlosBHgFWBkz+FoCq9hOReOB9ERkCtAV+AwxW1SIRmS4iN6jqEuA+4JCq9hCRScCvgYmXvKc1JPaypnxzRFe+OaIrn+85wtycAt5ZXcDiTXtp06IJt/dPYGxaIgM7t0HErk8Y0xCdOlvKvDVOkb3mDfOGlVBOK6UDeaq6DUBEZgCjgY0VlnsWeA74QcC0VGAJgKruE5HDgAdQYIuqFjnLLQbGOcuOBp5yps8GXhIR0TC8rap3x1Y8eWsrfnhzL5bl7WdOTgGZ3nz+tmIn3eJaMi4tiTsGJZLQgB6MMcb4i+wdbUBF9oIJJTkkAvkB4z5gaOACIjII6Kyq74pIYHLIBUY7CaUzMNj5+S+gt3PKyAeMAaIrbk9VS0SkGGgP7K+wzQeABwCSk5ND2I3aExUZwbW94rm2VzxHTp3lvbW7mZPj4zeLNvPbf27mqu7tGZeWxKi+HWkRbZd5jKnvZmb5n4sa1kCK7AUTyl+qYOdGzn2LF5EI4AXg3iDLTQOuALzATmA5UKKqh0TkQWAmUOZM7xbK9s5NUH0FeAX8zzmEsB91olWzJkxKT2ZSejI7Dxxnbk4Bc1f7+H5mLv9v3npu6duJcYMTGda1fYMp0GVMY5J/8ATLvzjAYyMvb9D/h0NJDj783/bLJQGFAeMxQF9gqXOOvSOwQEQyVNULPFq+oIgsB7YCqOpCYKEz/QGg/Gpu+fZ8IhIFtAYOVnvPwkCX9i15dOTlfO+Gnnh3HmJOto9/rPP3KhLbNHdui02kW9xlbodqjAnRLG9+gyuyF0woySEL6CkiXYECYBIwpXymqhYDseXjIrIU+IGqekWkBf6nsI+LyEj8vYaNznLxznWItsBDwARnFQuAe4DPgPHAv8LxekN1REQI6V3bkd61HU9l9OGfG/cwJ6eA/12ax0sf5TEouQ3j0pK4vX9Cg3iDlDENVWmZMjvbx1d6xjX4a4lVJgfnvP9UYBEQCUxT1Q0i8gzgVdUFF2geDywSkTL8ieXugHm/F5EBzvAzqrrFGX4d+JuI5OHvMUyq3i6Ft+bRkYwemMjogYnsPXLK/zR2jo//N289zyzcyI2p8YxLS+Irl8fRJLJR10U0Juwsc4rs/firqW6HUuustlIYUFU2FB5hdraPBbmFHDx+htjLoskYkMi4wYmkdmplt8UaEwYefiuH5Xn7WfHkDTSNinQ7nEtmtZXCnIjQN7E1fRNb8+StV/DvLUXMyfbxtxU7mPbpdnp3jGFcWhKjByUQH9PM7XCNaZQOHT/Dhxv28rVhyQ0iMVTFkkOYiY6KYGRqB0amduDQ8TO8u7aQ2TkF/Py9Tfzy/U185fI4xqUlMTK1A82aNPxfUGPCxTynyN7EIQ332YZAdlqpnsjbd4y5OT7eWV3A7uJTxDSL4rb+nRiXlsTgLm3ttJMxtUhVueX3nxAdFcGCqSPcDqfG2GmlBqBH/GX8cFRvHrupFyu2HWBOto95qwt5e1U+Xdq3YOygJMamJdK5nb2kyJiatr7gCJ/vOcrPGmiRvWAsOdQzkRHC8B6xDO8RyzNjnJcUZft4YfEWXli8hfSu7RiflsQt/ToSYy8pMqZGzPTuomlUBLc30CJ7wdhppQbCd+iEc1tsAdv3H6dZkwhu7tORcWlJDO8Ray8pMuYinTpbypCfL+bGKzrwwsSBbodTo+y0UiOQ1LYFU6/vycPX9WB1/mHmZPtYmFvI/DWFdGjVlDGDEhmflkTPDvaSImOq44P1/iJ7d3oa9hPRFVnPoQE7dbaUf32+jznZPpZuKaK0TOmX2JpxaYlkDEykXcvoqldiTCM3+ZUVFBw+ydIfXNvgailZz6GRatYkklv7deLWfp3Yf+w089cUMifbx1MLN/Lz9zZxXa94xqYlcX3veKKj7GlsYyradeAEn207wA9uathF9oKx5NBIxF7WlPtGdOW+EV3ZtPsIc3N8zFtTyD837qVtiyZkDEhgbFoS/ZNa222xxjhmZecT0QiK7AVjp5UasZLSMj7J28+cbB//3LiXMyVl9Ii/jLFpidwxKJFOrRt2YTFjLqS0TBnx63/Rq2MMb3wj3e1waoWdVjJBRUVGcF2veK7rFU/xybO8t243c7J9PPfBZn6zaDMjesQyNi2Rm/vYS4pM4/PJ1iJ2F5/iJ7c1/CJ7wdj/eANA6+ZNmJyezGTnJUVzcgqYm+Pj0Zm5tIxez639OjE2LYmhXds1unOvpnGa5fXRrmU0N17Rwe1QXGHJwVTSpX1Lvj/ycv7rhp5k7TjInBwf763bw6xsH0ltmzPR05mJQzoT38qKAJqG6eDxM/xz4x7uHpbSaG/WsORgzisiQhjarT1Du7Xn6Yy+/HPjHjK9+Tz/4RZeXLKVG6+IZ8rQLlzdI9Z6E6ZBmbe6gLOl2miK7AVjycGEJPAlRdv3H2fGql3MyvaxaMNeOrdrzqQhyUzwdCYupqnboRpzSVSVTG8+A5Ja06tj431otHH2l8wl6RrbkiduvYLPnrieP0weRGKb5vxm0Wau/OUSHnozm2Vb91NWVv/vgjON01pfMZ/vOcqERtxrgBCTg4iMEpHNIpInIo9fYLnxIqIi4nHGo0XkLyKyTkRyReTagGUnO9PXisgHIhLrTH9KRApEZI3zufUS99HUkqZRkWQMSGDGA1ey5LFruPeqFJZ/cYC7Xl/Jdc8v5c///oL9x067HaYx1ZLpzadZk8ZVZC+YKp9zEJFIYAswEvABWcBkVd1YYbkY4B9ANDBVVb0i8jDgUdVviEg88D4wBH9SKgRSVXW/iDwHnFDVp0TkKeCYqv421J2w5xzCx6mzpSzasIc3V+5i1faDNIkUbu7TkSlDk7myW3t7wM6EtZNnSkn/+WJGpnbgdw2syF4wl/qcQzqQp6rbnJXNAEYDGyss9yzwHPCDgGmpwBIAVd0nIocBD7AaEKCliBwAWgF5Ie+RCVvNmvzn2kTevqO8tTKfOTk+3l27m26xLZmcnsy4wUlW18mEpQ827Obo6ZJGf0oJQjutlAjkB4z7nGnniMggoLOqvluhbS4wWkSiRKQrMNhZ7izwILAOpwcBvB7QbqpzummaiLQNFpSIPCAiXhHxFhUVhbAbpq71iI/hf25PZeWTN/C7CQNo1zKan7+3iWG/WMIjb69mxbYDNIQn9E3DMTPL//KsoV3buR2K60JJDsHOA5z7Hy0iEcALwGNBlpuGP5l4gReB5UCJiDTBnxwGAQnAWuAJp82fgO7AQGA38HywoFT1FVX1qKonLi4uhN0wbmnWJJKxaUnMfvAqFv3XV5gyNJmPNu9j0isruPF3/+a1T7Zx+MQZt8M0jdzOA8dZse0gEzyd7fQnoSUHHxDYx0rC/22/XAzQF1gqIjuAYcACEfGoaomqPqqqA1V1NNAG2Ir/Dz+q+oX6vzpmAlc50/aqaqmqlgGv4j+tZRqIXh1jeCqjD6uevJHfjO9P6+ZN+Nk/NpH+iyU8OnMNWTsOWm/CuGKW1+cvspfW+IrsBRPKNYcsoKdzWqgAmARMKZ+pqsVAbPm4iCwFfuBckG6B/6L3cREZCZSo6kYRSQBSRSROVYvwX+ze5LTvpKq7ndXdAay/5L00Yad5dCR3ejpzp6czm3Yf4a2Vu5i3uoB3VhfQM/4ypgxNZuygJFq3sFedmtpXWqbMzvZxzeVxdGxtT/5DCMlBVUtEZCqwCIgEpqnqBhF5BvCq6oILNI8HFolIGf7EcrezzkIReRr4WETOAjuBe502z4nIQPynrnYA376oPTP1xhWdWvHsmL48cWtv3s3dzZurdvH0wo386v3Pua1/AlOGdiYtua119U2t+XhrEXuOnOKntzfOInvBWMluE5Y2FBbz1spdzF9TyLHTJfTuGMPk9GTuSEukVTPrTZia9dCb2azYdpAVT9zQqGopXehWVksOJqwdP13CgtxC3lq5i3UFxf6Hk/onMGVoMgM7t7HehLlkB46dZtgvl/D1K1MaXXlue5+DqbdaNo06V0p8na+Yt1btZP6aQmZl+7iiUyumDE1mzMAEYqw3YS7SvDWFnC1VJnjs2YZA1nMw9c7RU2eZv8bfm9i4+wgtov1lPKYMTaZ/Uhu3wzP1iKoy6sVPaBYdyfyHh7sdTp2znoNpUGKaNeGuYV342tBkcn3FvLXS35uYkZVP38RWTEnvQsbABC5rar/e5sJyfcVs3nuUX9zRz+1Qwo71HEyDcOTUWeavLuDNlbv4fM9RWkZHMnpQIlPSk+mb2Nrt8EyYevKddczN8bHqxzc2yhsdrOdgGrxWzZpw95Up3DWsC6vzD/PWyl3MzfHx1spd9E9qzZT0ZG4fkEBL600Yx8kzpSxcU8it/To1ysRQFfufYhoUESEtuS1pyW35yW2pvJPj461Vu3h87jp+9o9NjBmUwJT0LqQmtHI7VOOy99f7i+xNtAvRQVlyMA1W6+ZNuHd4V+65KoXsnYd4a+UuMr0+/r5iFwM7t2HK0GRu759A8+hIt0M1LpiZlU9K+xakW5G9oBrP0x6m0RIRPCnt+N3Egax68gZ+clsqR0+d5Yez15L+i8X8dP56Nu856naYpg7t2H+cldsPcqcV2Tsv6zmYRqVNi2juG9GVbw5PYdX2g7y1ahdvr8pn+mc7GdylLZPTk7mtfyeaNbHeREM2KzvfiuxVwe5WMo3eweNnzl283rb/OK2aRTE2LYmvDU2mZ4fG+4L5hqq0TLnqV0vok9CaafcOcTscV9ndSsZcQLuW0dx/dTfuG9GVFdv8vYk3V+7kjeU7GJLSlilDk7mlr/UmGoqPtxSx98hpns6wXsOFWHIwxiEiXNm9PVd2b8+BY6nMzvbx9qpdPDozl6cXbmRcWhKT05PpEX+Z26GaS5Dpzad9y2iu793B7VDCmiUHY4Jof1lTvn1Nd751dTc+23aAt1buYvryHby+bDtDu7ZjytBkRvXtSNMo603UJweOnWbxpr3cc2VKo6q+ejEsORhzARERwvAesQzvEUvR0dPMys5nxqp8vjdjDe1aRjN+sL830TW2pduhmhC8s7rAX2RviD3bUBW7IG1MNZWVKcvy9vPWyl18uGmv/wJn9/ZMTk/m5j4d7RtpmFJVbn7xY1pERzGvERbZC8YuSBtTgyIihK9cHsdXLo9j35FTZHrzeXtVPt99ezXtW0Yz3pPElPRkurS33kQ4WZN/mC17j/HLsVZkLxQhfcURkVEisllE8kTk8QssN15EVEQ8zni0iPxFRNaJSK6IXBuw7GRn+loR+UBEYp3p7UTkQxHZ6vxse4n7aEytiW/VjKnX9+TjH17HG98YwuAubXntk+1c85ul3PXaSj5Yv5uG0DtvCDK9Ppo3ieS2/p3cDqVeqDI5iEgk8DJwC5AKTBaRSq9LEpEY4BFgZcDkbwGoaj9gJPC8iESISBTwe+A6Ve0PrAWmOm0eB5aoak9giTNuTFiLjBCu7RXPK1/38OmPruf7Iy9nW9ExvvP3HB5+K4fjp0vcDrFRO3GmhIW5/iJ79mKo0ITSc0gH8lR1m6qeAWYAo4Ms9yzwHHAqYFoq/j/wqOo+4DDgAcT5tBT/s+utgEKnzWhgujM8HRhTnR0yxm0dWzfjkRt68smPrufJW3vzwfo9jHn5U7bvP+52aI3W++v2cOx0CRPtQnTIQkkOiUB+wLjPmXaOiAwCOqvquxXa5gKjRSRKRLoCg53lzgIPAuvwJ4VU4HWnTQdV3Q3g/IwPFpSIPCAiXhHxFhUVhbAbxtStyAjhga9056/fHMr+Y6fJeGkZSzbtdTusRmmmN5+usS0ZkmJnqUMVSnIIVpXq3ElUEYkAXgAeC7LcNPzJxAu8CCwHSkSkCf7kMAhIwH9a6YnqBK6qr6iqR1U9cXFx1WlqTJ0a0TOWBVNHkNyuBfdN9/L7xVspK7PrEHVl+/7jrNp+kDs9SVZkrxpCSQ4+ILAvlsR/TgEBxAB9gaUisgMYBiwQEY+qlqjqo6o6UFVHA22ArcBAAFX9Qv1X6zKBq5z17RWRTgDOz30XvXfGhInO7Vow58GrGJuWyAuLt/DA37I5cuqs22E1CrO8VmTvYoSSHLKAniRCUwcAABJLSURBVCLSVUSigUnAgvKZqlqsqrGqmqKqKcAKIENVvSLSQkRaAojISKBEVTcCBUCqiJR/5R8JbHKGFwD3OMP3APMvbReNCQ/NmkTy/J0DeOr2VJZu3seYlz5l614rFV6bSkrLmJPj47pe8XRo1cztcOqVKpODqpbgv5NoEf4/4JmqukFEnhGRjCqaxwM5IrIJ+BFwt7POQuBp4GMRWYu/J/ELp82vgJEishV/0vhV9XfLmPAkItw7vCtv3j+UI6fOMublT/lg/W63w2qwPt7qL7J3p73trdrsCWljXLKn+BTf+Xs2a/IP89C13Xnspl5ERtg58Zr0nb9l4915kM+euIEmkfbkekUXekLajpYxLunYuhkzvz2Myemd+d+lX/CNN7I4fOKM22E1GPudInt3DEq0xHAR7IgZ46KmUZH8cmx/fnFHPz77Yj+3v7SMjYVH3A6rQXgnp4CSMmWCnVK6KJYcjAkDU4YmM/PbV3KmpIyxf/qU+WsK3A6pXlNVMr35DEpuY2/zu0iWHIwJE2nJbVn43RH0S2zN92as4WfvbqSktMztsOql1fmH2brvGBOt13DRLDkYE0biY5rx5v3DuOfKLry2bDt3v76KA8dOux1WvTPLm+8vsjcgwe1Q6i1LDsaEmeioCJ4e3Zff3jmA7F2HuP2Py1jnK3Y7rHrDX2RvN1/t34nLmtpbCS6WJQdjwtT4wUnM+c5ViAjj/rycWd78qhsZ3rMiezXCkoMxYaxfUmsWTB2Op0tb/nv2Wv5n/nrOlNh1iAvJzMqnW2xLPF2syN6lsORgTJhrf1lT/vrNdL51dVf++tlOvvbaCvYdPVV1w0ZoW9ExVu04yJ2ezlZk7xJZcjCmHoiKjODHX03lD5MHsb7gCLf9YRnZOw+5HVbYmZXtIzJCGJeWWPXC5oIsORhTj2QMSGDuQ1fRrEkkk175jLdW7nI7pLBRUlrGnGwf1/WKI96K7F0ySw7G1DNXdGrFgqnDuap7LE++s47H56zl1NlSt8Ny3b+3FLHvqBXZqymWHIyph9q0iGbavUN4+LruzMjKZ+IrK9hdfNLtsFyV6c0n9rJoru8d9OWRpposORhTT0VGCP99c2/+fNdg8vYe5fY/LmPltgNuh+WKoqOnWbJpH2PTkqzIXg2xo2hMPTeqb0fmTx1Oq+ZN+NprK/nLp9tpCKX4q+Od1T6nyJ697a2mWHIwpgHoER/DvIeHc22veJ5euJHHMnM5eaZxXIfwF9nzkZbchh7xVmSvplhyMKaBaNWsCa/cPZjvj7ycd9YUMP7Py8k/eMLtsGpdzq7D5O07Zk9E17CQkoOIjBKRzSKSJyKPX2C58SKiIuJxxqNF5C8isk5EckXkWmd6jIisCfjsF5EXnXn3ikhRwLz7a2A/jWkUIiKER27oyev3eNh18AQZLy1j2db9bodVq2Z582kRHclX+1uRvZpUZXIQkUjgZeAWIBWYLCKpQZaLAR4BVgZM/haAqvbD/z7o50UkQlWPqurA8g+wE5gb0G5mwPzXLnbnjGmsru/dgQVTRxAX05SvT1vJ//37iwZ5HeL46RIW5hby1X5WZK+mhdJzSAfyVHWbqp4BZgCjgyz3LPAcEPhcfyqwBEBV9wGHgS+9r1REegLxwCfVjt4Yc15dY1vyzkPDGdW3I798/3Omvr2a46dL3A6rRr23bjfHz5TaKaVaEEpySAQCy0H6nGnniMggoLOqvluhbS4wWkSiRKQrMBio+K84GX9PIfBrzTgRWSsis0Uk6L+6iDwgIl4R8RYVFYWwG8Y0Pi2bRvHylDQev6U376/bzdj/Xc6O/cfdDqvGZHrz6RbXksFWZK/GhZIcglWvOveHXEQigBeAx4IsNw1/MvECLwLLgYpfXSYBbweMLwRSVLU/sBiYHiwoVX1FVT2q6omLiwthN4xpnESE71zTnenfTGfv0VNkvLSMjz7f53ZYl2xb0TGydhxighXZqxWhJAcfX/62nwQUBozHAH2BpSKyAxgGLBARj6qWqOqjzrWD0UAbYGt5QxEZAESpanb5NFU9oKrlr756FX9vwxhzia7uGcfCqSNIatuCb07P4g9LtlJWVn+vQ2R6/UX2xlqRvVoRSnLIAnqKSFcRicb/TX9B+UxVLVbVWFVNUdUUYAWQoapeEWkhIi0BRGQkUKKqGwPWPZkv9xoQkU4BoxnApovZMWNMZZ3btWDOg1cxekACv/twC9/+ezZHT511O6xqKyktY06Oj+t6xRMfY0X2akOVyUFVS4CpwCL8f6gzVXWDiDwjIhlVNI8HckRkE/Aj4O4K8ydQITkAj4jIBhHJxX/3071V74YxJlTNoyN5YeJA/ue2VP71+T5Gv/wpefuOuR1WtSzdXETR0dP2RHQtkoZwe5vH41Gv1+t2GMbUOyu2HeDhN3M4XVLG8xMGcHOfjm6HFJIH/uolZ9dhPnviequldAlEJFtVPcHm2VE1phEb1q09C787gu5xLfn237L57aLNlIb5dYiio6f51+f7GJeWaImhFtmRNaaRS2jTnJnfvpIJniRe+iiP+6ZnUXwifK9DzM3xF9mz9zbULksOxhiaNYnk1+P687Mxffk0bz8ZLy/j8z1H3A6rEn+RvXw8XdrSI/4yt8Np0Cw5GGMA//MQdw3rwowHruTkmVLueHk5C3MLq25Yh3J2HeKLouNMsF5DrbPkYIz5ksFd2vLud0eQmtCK7769ml+8t4mS0jK3wwIgM8vnFNnrVPXC5pJYcjDGVBLfqhlvf2sYdw/rwisfb+Oev6zi4PEzrsZ0/HQJ764t5Lb+nWhpRfZqnSUHY0xQ0VERPDumL8+N70/WjkPc/sdlrC8odi2ef1iRvTplycEYc0ETPJ2Z/Z0rUVXG/Wk5c7J9rsSRmeUvspeWbEX26oIlB2NMlfontWHBd0cwKLkNj83K5akFGzhbh9chvig6hnfnISZakb06Y8nBGBOS2Mua8vf7hnLfiK68sXwHX3t1JUVHT1fdsAZkevOJjBDusCJ7dcaSgzEmZFGREfzktlR+P2kgawsOc9sfPyFn16Fa3ebZ0jLmZBdwfW8rsleXLDkYY6pt9MBE5j44nOioCCb93wreXrWr1ra1dHMR+4+dtmcb6pglB2PMRUlNaMXCqSMY2q0dT8xdxxNz13G6pLTGtzMzK5+4mKZc18te6lWXLDkYYy5amxbRvPGNdB68tjtvr9rFpFdWsKf4VNUNQ7Tv6Ck+2ryPsWmJRFmRvTplR9sYc0kiI4QfjerNn76WxuY9R7ntj8tYtf1gjax7bk4BpWVqp5RcYMnBGFMjbunXiXkPDyemWRRTXl3B9OU7uJT3xZQX2RuS0pbucVZkr65ZcjDG1JjLO8Qw7+HhXHN5HD9dsIHHZuVy6uzFXYfI3nmIbUXHrTS3S0JKDiIySkQ2i0ieiDx+geXGi4iKiMcZjxaRv4jIOhHJFZFrnekxIrIm4LNfRF505jUVkZnOtlaKSMol76Uxps60bt6EV7/u4Xs39GRuTgHj/7wc36ET1V5PpjefltGRfLWfFdlzQ5XJQUQigZeBW4BUYLKIpAZZLgb/O59XBkz+FoCq9gNGAs+LSISqHlXVgeUfYCcw12lzH3BIVXsALwC/vui9M8a4IiJCeHTk5bz2dQ87958g46VPWZ63P+T2x06X8O7a3dzWP8GK7LkklJ5DOpCnqttU9QwwAxgdZLlngeeAwFsVUoElAKq6DzgMfOl9pSLSE4gHPnEmjQamO8OzgRvEnpc3pl66MbUD86cOp13LaO56fSWvfrwtpOsQ763dzYkzpUywInuuCSU5JAL5AeM+Z9o5IjII6Kyq71ZomwuMFpEoEekKDAYq/mtPBmbqf35jzm1PVUuAYqB9xaBE5AER8YqIt6ioKITdMMa4oVvcZcx7eDg3pXbk5+9t4pEZazhxpuSCbWZ68+ke15K05DZ1FKWpKJTkEOxb+7nULyIR+E//PBZkuWn4k4kXeBFYDlT8rZgEvB3q9s5NUH1FVT2q6omLs4djjAlnlzWN4k93pfHDUb14d20hY/93OTsPHA+6bN6+Y2TvPMTEIVZkz02hJAcfX/62nwQEvjswBugLLBWRHcAwYIGIeFS1RFUfda4tjAbaAFvLG4rIACBKVbODbU9EooDWQM3cNG2McY2I8NC1PXjjG+nsLj7F7X9cxtLN+yotN8ubT1SEcMegJBeiNOVCSQ5ZQE8R6Soi0fi/6S8on6mqxaoaq6opqpoCrAAyVNUrIi1EpCWAiIwESlR1Y8C6J/PlXgPOuu9xhscD/9JLuVnaGBNWrrk8joVTR5DQpjnfeCOLlz/KO3cd4mxpGXNy/EX24mKauhxp41ZlcnDO+08FFgGbgExV3SAiz4hIRhXN44EcEdkE/Ai4u8L8CVRODq8D7UUkD/g+cN5bZ40x9VNy+xbMfegqbu+fwG8WbeY7f8/m2OkSPvp8nxXZCxPSEL6Uezwe9Xq9bodhjKkmVeX1Zdv55fufk9K+BW1aRLPr4Ak+e/x6q6VUB0QkW1U9webZ0TfGuEZEuP/qbvztvnQOnThL9s5DjEtLssQQBuzpEmOM667qHsvC745g2rLt3Deiq9vhGCw5GGPCRGKb5vzktkrFF4xLrO9mjDGmEksOxhhjKrHkYIwxphJLDsYYYyqx5GCMMaYSSw7GGGMqseRgjDGmEksOxhhjKmkQtZVEpAj/q0YvRiwQ+vsL647FVT0WV/WFa2wWV/VcSlxdVDXoC3EaRHK4FCLiPV/hKTdZXNVjcVVfuMZmcVVPbcVlp5WMMcZUYsnBGGNMJZYc4BW3AzgPi6t6LK7qC9fYLK7qqZW4Gv01B2OMMZVZz8EYY0wllhyMMcZU0qCTg4iMEpHNIpInIo8HmX+viBSJyBrnc3/AvHtEZKvzuSeM4ioNmL6gLuNylpkgIhtFZIOIvBUw3bXjVUVcrh0vEXkhYNtbRORwwDw3f78uFJebxytZRD4SkdUislZEbg2Y94TTbrOI3BwOcYlIioicDDhef67juLqIyBInpqUikhQw79J/v1S1QX6ASOALoBsQDeQCqRWWuRd4KUjbdsA252dbZ7it23E58465eLx6AqvLjwUQHybHK2hcbh+vCst/F5gWDsfrfHG5fbzwX1h90BlOBXYEDOcCTYGuznoiwyCuFGC9i8drFnCPM3w98Lea/P1qyD2HdCBPVbep6hlgBjA6xLY3Ax+q6kFVPQR8CIwKg7hqUyhxfQt42TkmqOo+Z7rbx+t8cdWm6v47TgbedobdPl7ni6s2hRKXAq2c4dZAoTM8GpihqqdVdTuQ56zP7bhqUyhxpQJLnOGPAubXyO9XQ04OiUB+wLjPmVbROKdbNltEOlezbV3HBdBMRLwiskJExtRQTKHGdTlwuYh86mx/VDXauhEXuHu8AH/3H/833n9Vt20dxwXuHq+ngLtExAe8h79XE2pbN+IC6Oqcbvq3iFxdQzGFGlcuMM4ZvgOIEZH2IbatUkNODhJkWsX7dhcCKaraH1gMTK9GWzfiAkhW/6PyU4AXRaR7HcYVhf8UzrX4v3G+JiJtQmzrRlzg7vEqNwmYraqlF9G2ui4lLnD3eE0G3lDVJOBW4G8iEhFiWzfi2o3/eA0Cvg+8JSKtqBmhxPUD4BoRWQ1cAxQAJSG2rVJDTg4+IPAbdxIVuoOqekBVTzujrwKDQ23rUlyoaqHzcxuwFBhUV3E5y8xX1bNO934z/j/Krh6vC8Tl9vEqN4kvn7px+3idLy63j9d9QKaz/c+AZviLyrl9vILG5ZzmOuBMz8Z/jeDyuopLVQtVdayTnH7sTCsOcZ+qVhsXU8Lhg//b5Db83ebyCzp9KizTKWD4DmCF/ueCznb8F3PaOsPtwiCutkBTZzgW2MoFLjbWQlyjgOkB288H2ofB8TpfXK4eL2e5XsAOnAdOw+H36wJxuf379T5wrzN8Bf4/aAL04csXpLdRcxekLyWuuPI48F84Lqjj3/tYIMIZ/jnwTE3+fl3yToTzB38XcAv+jP5jZ9ozQIYz/Etgg3PgPwJ6B7T9Jv4LX3nAN8IhLuAqYJ0zfR1wXx3HJcDvgI3O9ieFyfEKGpfbx8sZfwr4VZC2rh2v88Xl9vHCf4H1U2f7a4CbAtr+2Gm3GbglHOLCf76//P9pDnB7Hcc1Hn8C3wK8hpPYa+r3y8pnGGOMqaQhX3MwxhhzkSw5GGOMqcSSgzHGmEosORhjjKnEkoMxxphKLDkYY4ypxJKDMcaYSv4/ii/VdX84JqsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid_search.cv_results_[ 'std_test_score' ]\n",
    "\n",
    "x_axis = colsample_bytree_s\n",
    "\n",
    "plt.plot(x_axis, -test_means)\n",
    "#plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = str(max_depths[i]) +' Test')\n",
    "#plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = str(max_depths[i]) +' Train')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### colsample_bytree=0.8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正则化参数lambda_l1(reg_alpha), lambda_l2(reg_lambda)感觉不用调了"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 减小学习率，调整n_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best n_estimators: 2613\n",
      "best cv score: 0.48511044419278376\n"
     ]
    }
   ],
   "source": [
    "params = {'boosting_type': 'goss',\n",
    "          'objective': 'multiclass',\n",
    "          'num_class':9, \n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.01,\n",
    "          #'n_estimators':n_estimators_1,\n",
    "          'max_depth': 7,\n",
    "          'num_leaves':90,\n",
    "          'min_child_samples':30,\n",
    "          'max_bin': 127, #2^6,原始特征为整数，很少超过100\n",
    "          'colsample_bytree': 0.8,\n",
    "         }\n",
    "n_estimators_2 = get_n_estimators(params , X_train , y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 用所有训练数据，采用最佳参数重新训练模型\n",
    "由于样本数目增多，模型复杂度稍微扩大一点？\n",
    "num_leaves增多5\n",
    "#min_child_samples按样本比例增加到40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(boosting_type='goss', class_weight=None, colsample_bytree=0.8,\n",
       "               importance_type='split', learning_rate=0.01, max_bin=127,\n",
       "               max_depth=7, min_child_samples=40, min_child_weight=0.001,\n",
       "               min_split_gain=0.0, n_estimators=2613, n_jobs=4, num_class=9,\n",
       "               num_leaves=95, objective='multiclass', random_state=None,\n",
       "               reg_alpha=0.0, reg_lambda=0.0, silent=False, subsample=1.0,\n",
       "               subsample_for_bin=200000, subsample_freq=0)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting_type': 'goss',\n",
    "          'objective': 'multiclass',\n",
    "          'num_class':9, \n",
    "          'n_jobs': 4,\n",
    "          'learning_rate': 0.01,\n",
    "          'n_estimators':n_estimators_2,\n",
    "          'max_depth': 7,\n",
    "          'num_leaves':95,\n",
    "          'min_child_samples':40,\n",
    "          'max_bin': 127, #2^6,原始特征为整数，很少超过100\n",
    "          'colsample_bytree': 0.8,\n",
    "         }\n",
    "\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "lg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 保存模型，用于后续测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump(lg, open(\"Otto_LightGBM_goss_tfidf.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 特征重要性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({\"columns\":list(feat_names), \"importance\":list(lg.feature_importances_.T)})\n",
    "df = df.sort_values(by=['importance'],ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>columns</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>feat_67_tfidf</td>\n",
       "      <td>37606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>feat_25_tfidf</td>\n",
       "      <td>35948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>feat_24_tfidf</td>\n",
       "      <td>34501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>feat_48_tfidf</td>\n",
       "      <td>32508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>feat_40_tfidf</td>\n",
       "      <td>29299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>feat_82_tfidf</td>\n",
       "      <td>1521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>feat_61_tfidf</td>\n",
       "      <td>1401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>feat_84_tfidf</td>\n",
       "      <td>1245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>feat_6_tfidf</td>\n",
       "      <td>885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>feat_51_tfidf</td>\n",
       "      <td>732</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>93 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          columns  importance\n",
       "66  feat_67_tfidf       37606\n",
       "24  feat_25_tfidf       35948\n",
       "23  feat_24_tfidf       34501\n",
       "47  feat_48_tfidf       32508\n",
       "39  feat_40_tfidf       29299\n",
       "..            ...         ...\n",
       "81  feat_82_tfidf        1521\n",
       "60  feat_61_tfidf        1401\n",
       "83  feat_84_tfidf        1245\n",
       "5    feat_6_tfidf         885\n",
       "50  feat_51_tfidf         732\n",
       "\n",
       "[93 rows x 2 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAVYUlEQVR4nO3df6xc5Z3f8fcnBhI2UWoIF+Tapma31m4IUgy5BbepKgoRGLKqWQm0oG2wECtvI6MmVdrG5B82JFRE6oYWKUHyLl7MKo2DSFZY4KxrEaJtpPDDBBYwDvIt0HBjF5vaENKoUNNv/5jnkokZ+47v73vn/ZJGd873POfc54zn+jPnOc/MpKqQJA229812ByRJs88wkCQZBpIkw0CShGEgSQJOmu0OTNQZZ5xRK1asmO1uSNK88uSTT75WVUNH1+dtGKxYsYJdu3bNdjckaV5J8j961R0mkiQZBpIkw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kS8/gdyJLmphUbH3r3/su3f3oWe6IT4ZmBJMkwkCQZBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEY6ASt2PjQb7ypSNLCYBhIksYPgyQfSPJ4kr9LsjvJl1v9niQvJXm63Va1epLcmWQkyTNJLuja17oke9ttXVf9E0mebdvcmSTTcbCSpN76+Wyit4BLquqXSU4GfpTk+23dv6uq+49qfwWwst0uAu4CLkpyOnALMAwU8GSSbVV1uLVZDzwKbAfWAN9HkjQjxj0zqI5ftsWT262Os8la4N623aPA4iRLgMuBnVV1qAXATmBNW/fhqvpxVRVwL3DVJI5JknSC+rpmkGRRkqeBA3T+Q3+srbqtDQXdkeT9rbYUeKVr89FWO159tEe9Vz/WJ9mVZNfBgwf76bokqQ99hUFVvVNVq4BlwIVJzgNuBn4P+EfA6cAXW/Ne4/01gXqvfmyqquGqGh4aGuqn65KkPpzQbKKqeh34IbCmqva3oaC3gL8ELmzNRoHlXZstA/aNU1/Woy5JmiH9zCYaSrK43T8V+BTw0zbWT5v5cxXwXNtkG3B9m1W0GnijqvYDO4DLkpyW5DTgMmBHW/dmktVtX9cDD0ztYWqixt5X4HsLpIWtn9lES4AtSRbRCY/7qurBJD9IMkRnmOdp4F+19tuBK4ER4FfADQBVdSjJV4AnWrtbq+pQu/9Z4B7gVDqziJxJJEkzaNwwqKpngPN71C85RvsCNhxj3WZgc4/6LuC88foiSZoevgNZkmQYSJIMA0kShoEkCcNAkoRhIEnCMJAkYRhIkujvHciSJqD7Izxevv3Ts9gTaXyeGUiSDANJkmEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkiT7CIMkHkjye5O+S7E7y5VY/J8ljSfYm+U6SU1r9/W15pK1f0bWvm1v9hSSXd9XXtNpIko1Tf5iSpOPp58zgLeCSqvo4sApYk2Q18DXgjqpaCRwGbmztbwQOV9U/BO5o7UhyLnAt8DFgDfDNJIuSLAK+AVwBnAtc19pKkmbIuGFQHb9siye3WwGXAPe3+hbgqnZ/bVumrb80SVp9a1W9VVUvASPAhe02UlUvVtXbwNbWVpI0Q/q6ZtBewT8NHAB2Av8deL2qjrQmo8DSdn8p8ApAW/8G8JHu+lHbHKsuSZohfYVBVb1TVauAZXReyX+0V7P2M8dYd6L190iyPsmuJLsOHjw4fsclSX05odlEVfU68ENgNbA4ydhHYC8D9rX7o8BygLb+7wGHuutHbXOseq/fv6mqhqtqeGho6ES6Lkk6jn5mEw0lWdzunwp8CtgDPAJc3ZqtAx5o97e1Zdr6H1RVtfq1bbbROcBK4HHgCWBlm510Cp2LzNum4uAkSf3p58ttlgBb2qyf9wH3VdWDSZ4Htib5KvAUcHdrfzfwV0lG6JwRXAtQVbuT3Ac8DxwBNlTVOwBJbgJ2AIuAzVW1e8qOUDrK2JfO+IUz0q+NGwZV9Qxwfo/6i3SuHxxd/z/ANcfY123AbT3q24HtffRXkjQNfAeyJMkwkKSpsmLjQ7/x3dfziWEgSTIMJEmGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBnPefP6yDEnzx7hhkGR5kkeS7EmyO8nnWv1Pk/w8ydPtdmXXNjcnGUnyQpLLu+prWm0kycau+jlJHkuyN8l3kpwy1QcqSTq2fs4MjgBfqKqPAquBDUnObevuqKpV7bYdoK27FvgYsAb4ZpJFSRYB3wCuAM4Fruvaz9favlYCh4Ebp+j4JEl9GDcMqmp/Vf2k3X8T2AMsPc4ma4GtVfVWVb0EjAAXtttIVb1YVW8DW4G1SQJcAtzftt8CXDXRA5IknbgTumaQZAVwPvBYK92U5Jkkm5Oc1mpLgVe6NhtttWPVPwK8XlVHjqr3+v3rk+xKsuvgwYMn0nVJ0nH0HQZJPgR8F/h8Vf0CuAv4HWAVsB/4s7GmPTavCdTfW6zaVFXDVTU8NDTUb9clSeM4qZ9GSU6mEwTfqqrvAVTVq13r/xx4sC2OAsu7Nl8G7Gv3e9VfAxYnOamdHXS3lyTNgH5mEwW4G9hTVV/vqi/pavYHwHPt/jbg2iTvT3IOsBJ4HHgCWNlmDp1C5yLztqoq4BHg6rb9OuCByR2WJOlE9HNm8EngM8CzSZ5utS/RmQ20is6QzsvAnwBU1e4k9wHP05mJtKGq3gFIchOwA1gEbK6q3W1/XwS2Jvkq8BSd8JEkzZBxw6CqfkTvcf3tx9nmNuC2HvXtvbarqhfpzDaSJM0C34EsSTIMJEmGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiT6/KA6SYOj+2tWX77907PYE80kzwwkSYaBJMkwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkkQfYZBkeZJHkuxJsjvJ51r99CQ7k+xtP09r9SS5M8lIkmeSXNC1r3Wt/d4k67rqn0jybNvmziS9vnNZkjRN+jkzOAJ8oao+CqwGNiQ5F9gIPFxVK4GH2zLAFcDKdlsP3AWd8ABuAS4CLgRuGQuQ1mZ913ZrJn9oC9uKjQ+9e5OkyRo3DKpqf1X9pN1/E9gDLAXWAltasy3AVe3+WuDe6ngUWJxkCXA5sLOqDlXVYWAnsKat+3BV/biqCri3a1+S5hBfgCxcJ3TNIMkK4HzgMeCsqtoPncAAzmzNlgKvdG022mrHq4/2qPf6/euT7Eqy6+DBgyfSdUnScfQdBkk+BHwX+HxV/eJ4TXvUagL19xarNlXVcFUNDw0NjddlSVKf+gqDJCfTCYJvVdX3WvnVNsRD+3mg1UeB5V2bLwP2jVNf1qMuSZoh/cwmCnA3sKeqvt61ahswNiNoHfBAV/36NqtoNfBGG0baAVyW5LR24fgyYEdb92aS1e13Xd+1L0nSDOjny20+CXwGeDbJ0632JeB24L4kNwI/A65p67YDVwIjwK+AGwCq6lCSrwBPtHa3VtWhdv+zwD3AqcD3202aUX6piwbZuGFQVT+i97g+wKU92hew4Rj72gxs7lHfBZw3Xl8kSdPDdyBLkgwDSVJ/1wwkSZMwH65HGQaa8+bDH5I03zlMpDnJjz2QZpZhIEkyDOYTXy1Lmi6GgSTJMJAkGQaSJAwDSRKGgSQJw0CaN5xNpulkGEiSDANJkmEgScIwkCRhGEiS6CMMkmxOciDJc121P03y8yRPt9uVXetuTjKS5IUkl3fV17TaSJKNXfVzkjyWZG+S7yQ5ZSoPUJI0vn7ODO4B1vSo31FVq9ptO0CSc4FrgY+1bb6ZZFGSRcA3gCuAc4HrWluAr7V9rQQOAzdO5oAkzQ9jU2WdLjs3jBsGVfW3wKE+97cW2FpVb1XVS8AIcGG7jVTVi1X1NrAVWJskwCXA/W37LcBVJ3gMkqRJmsw3nd2U5HpgF/CFqjoMLAUe7Woz2moArxxVvwj4CPB6VR3p0f49kqwH1gOcffbZk+i6NNj89jgdbaIXkO8CfgdYBewH/qzV06NtTaDeU1VtqqrhqhoeGho6sR5Lko5pQmcGVfXq2P0kfw482BZHgeVdTZcB+9r9XvXXgMVJTmpnB93tJUkzZEJnBkmWdC3+ATA202gbcG2S9yc5B1gJPA48AaxsM4dOoXOReVtVFfAIcHXbfh3wwET6pPnPi4nS7Bn3zCDJt4GLgTOSjAK3ABcnWUVnSOdl4E8Aqmp3kvuA54EjwIaqeqft5yZgB7AI2FxVu9uv+CKwNclXgaeAu6fs6GaBY7GS5qNxw6CqrutRPuZ/2FV1G3Bbj/p2YHuP+ot0ZhtJkmaJ70CWNOc4ZDjzDANJkmEgSTIMpGNyqEKDxDCQJBkGkiTDQJKEYSBJwjCQJGEYSJKY3PcZaBx+TpGk+cIzA81rvhdg4fDfcnYZBpIkh4kkTd6gvqJfSEPBhoEkzUEzHTQOEwlwvFYadIaBpHnLFzFTxzCQJHnNYLaMvZqZzxedFtLFM2nQjXtmkGRzkgNJnuuqnZ5kZ5K97edprZ4kdyYZSfJMkgu6tlnX2u9Nsq6r/okkz7Zt7kySqT7IQeOps6QT1c8w0T3AmqNqG4GHq2ol8HBbBrgCWNlu64G7oBMewC3ARcCFwC1jAdLarO/a7ujfJUnzztiLsvnywmzcMKiqvwUOHVVeC2xp97cAV3XV762OR4HFSZYAlwM7q+pQVR0GdgJr2roPV9WPq6qAe7v2JUmaIRO9ZnBWVe0HqKr9Sc5s9aXAK13tRlvtePXRHvWekqyncxbB2WefPcGuS/OD12Q0k6b6AnKv8f6aQL2nqtoEbAIYHh4+ZrtB438akiZrolNLX21DPLSfB1p9FFje1W4ZsG+c+rIedUnSDJpoGGwDxmYErQMe6Kpf32YVrQbeaMNJO4DLkpzWLhxfBuxo695MsrrNIrq+a1+SpBky7jBRkm8DFwNnJBmlMyvoduC+JDcCPwOuac23A1cCI8CvgBsAqupQkq8AT7R2t1bV2EXpz9KZsXQq8P12k7RALIT31AyCccOgqq47xqpLe7QtYMMx9rMZ2Nyjvgs4b7x+SJKmj+9AlvrgRfq5z3+jyTEMtCAd/R+DQxXS8flBdZIkw0CS5DDRQFnoY6rz5TNgpLloIMNgof+nKEknaiDDQBrj2YTUYRhIA8Lg0/EYBnOMQ1iDYTqnujqNVhNhGGheMSyl6eHUUkmSZwbSfOQZkqaaZwaSJM8MpPnOs4TpN9UX5efiRX7DQNK8YfBNH8NA0pzm+yNmhtcMJEmGgaTBsmLjQ3PqbGOu9GdSYZDk5STPJnk6ya5WOz3JziR728/TWj1J7kwykuSZJBd07Wdda783ybrJHdLUmSv/SJI03abimsE/r6rXupY3Ag9X1e1JNrblLwJXACvb7SLgLuCiJKcDtwDDQAFPJtlWVYenoG9zhhe+pLnHv8tfm44LyGuBi9v9LcAP6YTBWuDeqirg0SSLkyxpbXdW1SGAJDuBNcC3p6Fv0rSai1MGp8tEjtUz7blrstcMCvivSZ5Msr7Vzqqq/QDt55mtvhR4pWvb0VY7Vv09kqxPsivJroMHD06y65KkMZM9M/hkVe1LciawM8lPj9M2PWp1nPp7i1WbgE0Aw8PDPdtMhqeM02OQXi1L89WkwqCq9rWfB5L8NXAh8GqSJVW1vw0DHWjNR4HlXZsvA/a1+sVH1X84mX5J6jCI59eLvNkcRptwGCT5IPC+qnqz3b8MuBXYBqwDbm8/H2ibbANuSrKVzgXkN1pg7AD+w9iso7afmyfaL81t8+kPcyoN6nFr/pjMmcFZwF8nGdvPf6mqv0nyBHBfkhuBnwHXtPbbgSuBEeBXwA0AVXUoyVeAJ1q7W8cuJkuSZsaEw6CqXgQ+3qP+v4BLe9QL2HCMfW0GNk+0L5p9g/zK16EYLQS+A1mSZBhIkvzU0jlhtmYQOLwhaYxhoPfwXaIaM8jXggaNYTAFfIWtqWIQTw0fxxNnGPRpvr5Cmo0/ivn6WPVroR+fBpMXkCVJhoEkzQfT/f0qDhNJEg7/DXwYeKFJkgwDzaJBfyUmzSVeM5AkGQaSJMNAknqa7tk7c41hIGng/uPTe3kBWZJOwEINTc8MJEmeGUyEUyIlLTSGgTTLFuqwg+aXOTNMlGRNkheSjCTZONv9kaRBMifCIMki4BvAFcC5wHVJzp3dXknS4JgTYQBcCIxU1YtV9TawFVg7y32SpIGRqprtPpDkamBNVf1xW/4McFFV3XRUu/XA+rb4u8ALk/i1ZwCvTWL7hcLHwcdgjI9Dx0J/HP5BVQ0dXZwrF5DTo/aelKqqTcCmKfmFya6qGp6Kfc1nPg4+BmN8HDoG9XGYK8NEo8DyruVlwL5Z6oskDZy5EgZPACuTnJPkFOBaYNss90mSBsacGCaqqiNJbgJ2AIuAzVW1e5p/7ZQMNy0APg4+BmN8HDoG8nGYExeQJUmza64ME0mSZpFhIEkazDAYxI++SLI8ySNJ9iTZneRzrX56kp1J9rafp812X2dCkkVJnkryYFs+J8lj7XH4TpvIsGAlWZzk/iQ/bc+JfzyIz4Uk/6b9PTyX5NtJPjBoz4UxAxcGA/zRF0eAL1TVR4HVwIZ23BuBh6tqJfBwWx4EnwP2dC1/DbijPQ6HgRtnpVcz5z8Df1NVvwd8nM5jMVDPhSRLgX8NDFfVeXQmr1zL4D0XgAEMAwb0oy+qan9V/aTdf5POH/9SOse+pTXbAlw1Oz2cOUmWAZ8G/qItB7gEuL81WdCPQ5IPA/8MuBugqt6uqtcZwOcCnRmVpyY5CfgtYD8D9FzoNohhsBR4pWt5tNUGRpIVwPnAY8BZVbUfOoEBnDl7PZsx/wn498D/a8sfAV6vqiNteaE/J34bOAj8ZRsq+4skH2TAngtV9XPgPwI/oxMCbwBPMljPhXcNYhj09dEXC1WSDwHfBT5fVb+Y7f7MtCS/Dxyoqie7yz2aLuTnxEnABcBdVXU+8L9Z4ENCvbRrImuBc4C/D3yQzvDx0Rbyc+FdgxgGA/vRF0lOphME36qq77Xyq0mWtPVLgAOz1b8Z8kngXyR5mc4Q4SV0zhQWt6ECWPjPiVFgtKoea8v30wmHQXsufAp4qaoOVtX/Bb4H/BMG67nwrkEMg4H86Is2Ln43sKeqvt61ahuwrt1fBzww032bSVV1c1Utq6oVdP7tf1BVfwQ8Alzdmi3ox6Gq/ifwSpLfbaVLgecZsOcCneGh1Ul+q/19jD0OA/Nc6DaQ70BOciWdV4NjH31x2yx3adol+afAfwOe5ddj5V+ic93gPuBsOn8c11TVoVnp5AxLcjHwb6vq95P8Np0zhdOBp4B/WVVvzWb/plOSVXQuoJ8CvAjcQOfF4UA9F5J8GfhDOrPtngL+mM41goF5LowZyDCQJP2mQRwmkiQdxTCQJBkGkiTDQJKEYSBJwjCQJGEYSJKA/w9fLy8BDGbNIQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.bar(range(len(lg.feature_importances_)), lg.feature_importances_)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finish\n"
     ]
    }
   ],
   "source": [
    "test = pd.read_csv(\"./Otto_FE_test_tfidf.csv\")\n",
    "test_id = test['id']   \n",
    "X_test = test.drop([\"id\"], axis=1)\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_test.columns \n",
    "\n",
    "#sklearn的学习器大多之一稀疏数据输入，模型训练会快很多\n",
    "#查看一个学习器是否支持稀疏数据，可以看fit函数是否支持: X: {array-like, sparse matrix}.\n",
    "#可自行用timeit比较稠密数据和稀疏数据的训练时间\n",
    "from scipy.sparse import csr_matrix\n",
    "X_test = csr_matrix(X_test)\n",
    "\n",
    "print('finish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finish\n"
     ]
    }
   ],
   "source": [
    "#load训练好的模型\n",
    "import pickle\n",
    "\n",
    "model = pickle.load(open(\"Otto_LightGBM_goss_org_tfidf.pkl\", 'rb'))\n",
    "\n",
    "#输出每类的概率\n",
    "y_test_pred = model.predict_proba(X_test)\n",
    "print('finish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(144368, 9)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#生成提交结果\n",
    "out_df = pd.DataFrame(y_test_pred)\n",
    "\n",
    "columns = np.empty(9, dtype=object)\n",
    "for i in range(9):\n",
    "    columns[i] = 'Class_' + str(i+1)\n",
    "\n",
    "out_df.columns = columns\n",
    "\n",
    "out_df = pd.concat([test_id,out_df], axis = 1)\n",
    "out_df.to_csv(\"LightGBM_goss_tfidf.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
